From cdd3d688d3ece6313fcc012e3a242ea74f46f67d Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 26 Apr 2024 09:49:42 -0400 Subject: [PATCH 001/203] Initial work on refactor modeling skeleton --- .../ExperimentImportService.java | 2 + .../experiment/ExperimentProcessor.java | 42 +++++ .../experiment/ExperimentUtilities.java | 20 +++ .../AppendOverwritePhenotypesWorkflow.java | 25 +++ .../experiment/create/model/ExistingData.java | 18 +++ .../steps/GetExistingProcessingStep.java | 148 ++++++++++++++++++ .../experiment/create/steps/ProcessStep.java | 15 ++ .../workflow/CreateNewExperimentWorkflow.java | 44 ++++++ .../experiment/model/ImportContext.java | 28 ++++ .../experiment/model/ProcessedData.java | 11 ++ .../experiment/pipeline/Pipeline.java | 18 +++ .../experiment/pipeline/ProcessingStep.java | 5 + .../workflow/ExperimentWorkflowFactory.java | 71 +++++++++ .../experiment/workflow/Workflow.java | 10 ++ 14 files changed, 457 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentProcessor.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ExistingData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ImportContext.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/Pipeline.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/ProcessingStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/Workflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index cd795564a..b7d9248d6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -71,6 +71,8 @@ public ImportPreviewResponse process(List brAPIImports, Table data, ImportPreviewResponse response = null; List processors = List.of(experimentProcessorProvider.get()); + Processor experimentProcessor = experimentProcessorProvider.get(); + //response = experimentProcessor.process(upload, brAPIImports, data, program, user, commit); response = processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); return response; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentProcessor.java new file mode 100644 index 000000000..8207086fe --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentProcessor.java @@ -0,0 +1,42 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.ExperimentWorkflowFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.Workflow; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.services.exceptions.ValidatorException; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.Map; + +@Slf4j +@Prototype +public class ExperimentProcessor { + + private final ExperimentWorkflowFactory experimentWorkflowFactory; + + @Inject + public ExperimentProcessor(ExperimentWorkflowFactory experimentWorkflowFactory) { + this.experimentWorkflowFactory = experimentWorkflowFactory; + } + + public Map process(ImportContext context) + throws ApiException, ValidatorException, MissingRequiredInfoException, UnprocessableEntityException { + + // determine which workflow to use based on the import context + Workflow workflow = experimentWorkflowFactory.getWorkflow(context); + log.info("Importing experiment data using workflow: " + workflow.getName()); + + ProcessedData output = workflow.process(context); + + + return new HashMap<>(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java new file mode 100644 index 000000000..bd90ee222 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -0,0 +1,20 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment; + +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; + +import java.util.List; +import java.util.stream.Collectors; + +public class ExperimentUtilities { + + public static final CharSequence COMMA_DELIMITER = ","; + + public static List importRowsToExperimentObservations(List importRows) { + return importRows.stream() + .map(trialImport -> (ExperimentObservation) trialImport) + .collect(Collectors.toList()); + } + + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java new file mode 100644 index 000000000..970fc25f3 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -0,0 +1,25 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.Workflow; + +@Prototype +public class AppendOverwritePhenotypesWorkflow implements Workflow { + + @Override + public ProcessedData process(ImportContext context) { + + // validate that all rows have an ObsUnitID + + // TODO: implement + return new ProcessedData(); + } + + @Override + public String getName() { + return "AppendOverwritePhenotypesWorkflow"; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ExistingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ExistingData.java new file mode 100644 index 000000000..99113cc9f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ExistingData.java @@ -0,0 +1,18 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; + +import lombok.*; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; + +import java.util.Map; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ExistingData { + private Map> observationUnitByNameNoScope; + // TODO: add rest of fields +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java new file mode 100644 index 000000000..fc69c8105 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -0,0 +1,148 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.steps; + +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ExistingData; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Prototype +@Slf4j +public class GetExistingProcessingStep implements ProcessingStep { + + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public GetExistingProcessingStep(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + } + + @Override + public ExistingData process(ImportContext input) { + + List experimentImportRows = ExperimentUtilities.importRowsToExperimentObservations(input.getImportRows()); + Program program = input.getProgram(); + + // getExisting + Map> observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); + // TODO: populate rest of data + + ExistingData existing = ExistingData.builder().observationUnitByNameNoScope(observationUnitByNameNoScope).build(); + + return existing; + } + + /** + * Initializes the observation units for the given program and experimentImportRows. + * + * @param program The program object + * @param experimentImportRows A list of ExperimentObservation objects + * @return A map of Observation Unit IDs to PendingImportObject objects + * + * @throws InternalServerException + * @throws IllegalStateException + */ + private Map> initializeObservationUnits(Program program, List experimentImportRows) { + Map> observationUnitByName = new HashMap<>(); + + Map rowByObsUnitId = new HashMap<>(); + experimentImportRows.forEach(row -> { + if (StringUtils.isNotBlank(row.getObsUnitID())) { + if(rowByObsUnitId.containsKey(row.getObsUnitID())) { + throw new IllegalStateException("ObsUnitId is repeated: " + row.getObsUnitID()); + } + rowByObsUnitId.put(row.getObsUnitID(), row); + } + }); + + try { + List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); + + // TODO: grab from externalReferences + /* + observationUnitByObsUnitId = existingObsUnits.stream() + .collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, + (BrAPIObservationUnit unit) -> new PendingImportObject<>(unit, false))); + */ + + String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + if (existingObsUnits.size() == rowByObsUnitId.size()) { + existingObsUnits.forEach(brAPIObservationUnit -> { + processAndCacheObservationUnit(brAPIObservationUnit, refSource, program, observationUnitByName, rowByObsUnitId); + + BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) + .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); + + ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); + row.setExpTitle(Utilities.removeProgramKey(brAPIObservationUnit.getTrialName(), program.getKey())); + row.setEnv(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getStudyName(), program.getKey())); + row.setEnvLocation(Utilities.removeProgramKey(brAPIObservationUnit.getLocationName(), program.getKey())); + }); + } else { + List missingIds = new ArrayList<>(rowByObsUnitId.keySet()); + missingIds.removeAll(existingObsUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList())); + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + + return observationUnitByName; + } catch (ApiException e) { + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + /** + * Adds a new map entry to observationUnitByName based on the brAPIObservationUnit passed in and sets the + * expUnitId in the rowsByObsUnitId map. + * + * @param brAPIObservationUnit the BrAPI observation unit object + * @param refSource the reference source + * @param program the program object + * @param observationUnitByName the map of observation units by name (will be modified in place) + * @param rowByObsUnitId the map of rows by observation unit ID (will be modified in place) + * + * @throws InternalServerException + */ + private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservationUnit, String refSource, Program program, + Map> observationUnitByName, + Map rowByObsUnitId) { + BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) + .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); + + ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); + row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); + observationUnitByName.put(createObservationUnitKey(row), + new PendingImportObject<>(ImportObjectState.EXISTING, + brAPIObservationUnit, + UUID.fromString(idRef.getReferenceId()))); + } + + private String createObservationUnitKey(ExperimentObservation importRow) { + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); + } + + private String createObservationUnitKey(String studyName, String obsUnitName) { + return studyName + obsUnitName; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java new file mode 100644 index 000000000..fa483a7c4 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java @@ -0,0 +1,15 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.steps; + +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ExistingData; + +public class ProcessStep implements ProcessingStep { + + @Override + public ProcessedData process(ExistingData input) { + + // TODO: implement + return new ProcessedData(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java new file mode 100644 index 000000000..06238aaab --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -0,0 +1,44 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.steps.GetExistingProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.steps.ProcessStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.Pipeline; +import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.Workflow; + +import javax.inject.Inject; +import javax.inject.Provider; + +@Prototype +@Slf4j +public class CreateNewExperimentWorkflow implements Workflow { + + private final Provider getExistingStepProvider; + private final Provider processStepProvider; + + @Inject + public CreateNewExperimentWorkflow(Provider getExistingStepProvider, + Provider processStepProvider) { + this.getExistingStepProvider = getExistingStepProvider; + this.processStepProvider = processStepProvider; + } + + @Override + public ProcessedData process(ImportContext context) { + + Pipeline pipeline = new Pipeline<>(getExistingStepProvider.get()) + .addProcessingStep(processStepProvider.get()); + ProcessedData processed = pipeline.execute(context); + + // TODO: return actual data + return processed; + } + + @Override + public String getName() { + return "CreateNewExperimentWorkflow"; + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ImportContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ImportContext.java new file mode 100644 index 000000000..0738d6a1e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ImportContext.java @@ -0,0 +1,28 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import tech.tablesaw.api.Table; + +import java.util.List; +import java.util.Map; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ImportContext { + private ImportUpload upload; + private List importRows; + private Map mappedBrAPIImport; + private Table data; + private Program program; + private User user; + private boolean commit; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java new file mode 100644 index 000000000..4c8fe464d --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java @@ -0,0 +1,11 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import lombok.*; + +@Getter +@Setter +@Builder +@ToString +@NoArgsConstructor +public class ProcessedData { +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/Pipeline.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/Pipeline.java new file mode 100644 index 000000000..09b657a89 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/Pipeline.java @@ -0,0 +1,18 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.pipeline; + +public class Pipeline { + + private final ProcessingStep currentStep; + + public Pipeline(ProcessingStep currentStep) { + this.currentStep = currentStep; + } + + public Pipeline addProcessingStep(ProcessingStep newStep) { + return new Pipeline<>(input -> newStep.process(currentStep.process(input))); + } + + public O execute(I input) { + return currentStep.process(input); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/ProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/ProcessingStep.java new file mode 100644 index 000000000..2407e646e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/ProcessingStep.java @@ -0,0 +1,5 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.pipeline; + +public interface ProcessingStep { + O process(I input); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java new file mode 100644 index 000000000..7f0b741fe --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java @@ -0,0 +1,71 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.workflow; + +import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.AppendOverwritePhenotypesWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.CreateNewExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.List; + +@Singleton +public class ExperimentWorkflowFactory { + + private final Provider createNewExperimentWorkflowProvider; + private final Provider appendOverwritePhenotypesWorkflowProvider; + + @Inject + public ExperimentWorkflowFactory(Provider createNewExperimentWorkflowProvider, + Provider appendOverwritePhenotypesWorkflowProvider) { + this.createNewExperimentWorkflowProvider = createNewExperimentWorkflowProvider; + this.appendOverwritePhenotypesWorkflowProvider = appendOverwritePhenotypesWorkflowProvider; + } + + /** + * Retrieves the appropriate workflow based on the provided import context. Validation will be done + * in selected workflow, not here. For example will not check if file has ObsUnitIDs that all rows have one. + * We are just checking the basic condition for what type of workflow to return. + * + * @param context import context containing import rows + * @return the workflow to be used for processing the import rows + */ + public Workflow getWorkflow(ImportContext context) { + + List importRows = context.getImportRows(); + + boolean hasExpUnitObsUnitIDs = importRows.stream() + .anyMatch(row -> { + ExperimentObservation expRow = (ExperimentObservation) row; + return StringUtils.isNotBlank(expRow.getObsUnitID()); + }); + + if (hasExpUnitObsUnitIDs) { + long distinctCount = importRows.stream() + .map(row -> { + ExperimentObservation expRow = (ExperimentObservation) row; + return expRow.getObsUnitID(); + }) + .distinct() + .count(); + + if (distinctCount != importRows.size()) { + // If have ExpUnit ObsUnitIDs and there are duplicates -> Append / Update SubObsUnit Phenotypes + // TODO: different workflow for subobs units? + return appendOverwritePhenotypesWorkflowProvider.get(); + } else { + // If have ExpUnit ObsUnitIDs and all are unique -> Append / Update ExpUnit Phenotypes + return appendOverwritePhenotypesWorkflowProvider.get(); + } + + } else { + // No ObsUnitIDs so creating experiment or appending env + return createNewExperimentWorkflowProvider.get(); + // TODO: different workflow for appending envs? Would have a dependency on DAO to check for existing trial name + } + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/Workflow.java new file mode 100644 index 000000000..6c0384a0f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/Workflow.java @@ -0,0 +1,10 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.workflow; + +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; + +public interface Workflow { + + ProcessedData process(ImportContext context); + String getName(); +} From d8ca03202472caa806a89ab141c315cc8c971f1d Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 29 Apr 2024 11:57:35 -0400 Subject: [PATCH 002/203] Added trials pending data to get existing step --- .../experiment/create/model/ExistingData.java | 18 -- .../experiment/create/model/PendingData.java | 28 +++ .../steps/GetExistingProcessingStep.java | 193 +++++++++++++++++- .../experiment/create/steps/ProcessStep.java | 6 +- 4 files changed, 218 insertions(+), 27 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ExistingData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ExistingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ExistingData.java deleted file mode 100644 index 99113cc9f..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ExistingData.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; - -import lombok.*; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; - -import java.util.Map; - -@Getter -@Setter -@Builder -@ToString -@AllArgsConstructor -@NoArgsConstructor -public class ExistingData { - private Map> observationUnitByNameNoScope; - // TODO: add rest of fields -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java new file mode 100644 index 000000000..c3f769a71 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -0,0 +1,28 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; + +import lombok.*; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.ProgramLocation; + +import java.util.HashMap; +import java.util.Map; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class PendingData { + private Map> observationUnitByNameNoScope; + private Map> trialByNameNoScope; + private Map> studyByNameNoScope; + private Map> locationByName; + private Map> obsVarDatasetByName; + private Map> existingGermplasmByGID; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java index fc69c8105..51997347c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -7,8 +7,12 @@ import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -16,7 +20,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ExistingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -26,29 +30,40 @@ @Prototype @Slf4j -public class GetExistingProcessingStep implements ProcessingStep { +public class GetExistingProcessingStep implements ProcessingStep { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private final BrAPITrialDAO brAPITrialDAO; + private final BrAPIStudyDAO brAPIStudyDAO; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @Inject - public GetExistingProcessingStep(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + public GetExistingProcessingStep(BrAPIObservationUnitDAO brAPIObservationUnitDAO, + BrAPITrialDAO brAPITrialDAO, + BrAPIStudyDAO brAPIStudyDAO) { this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + this.brAPITrialDAO = brAPITrialDAO; + this.brAPIStudyDAO = brAPIStudyDAO; } @Override - public ExistingData process(ImportContext input) { + public PendingData process(ImportContext input) { List experimentImportRows = ExperimentUtilities.importRowsToExperimentObservations(input.getImportRows()); Program program = input.getProgram(); - // getExisting + // Populate pending objects with existing status Map> observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); + Map> trialByNameNoScope = initializeTrialByNameNoScope(program, observationUnitByNameNoScope, experimentImportRows); + // TODO: populate rest of data - ExistingData existing = ExistingData.builder().observationUnitByNameNoScope(observationUnitByNameNoScope).build(); + PendingData existing = PendingData.builder() + .observationUnitByNameNoScope(observationUnitByNameNoScope) + .trialByNameNoScope(trialByNameNoScope) + .build(); return existing; } @@ -145,4 +160,170 @@ private String createObservationUnitKey(ExperimentObservation importRow) { private String createObservationUnitKey(String studyName, String obsUnitName) { return studyName + obsUnitName; } + + /** + * Initializes trials by name without scope for the given program. + * + * @param program the program to initialize trials for + * @param observationUnitByNameNoScope a map of observation units by name without scope + * @param experimentImportRows a list of experiment observation rows + * @return a map of trials by name with pending import objects + * + * @throws InternalServerException + */ + private Map> initializeTrialByNameNoScope(Program program, Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> trialByName = new HashMap<>(); + + initializeTrialsForExistingObservationUnits(program, observationUnitByNameNoScope, trialByName); + + List uniqueTrialNames = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID())) + .map(ExperimentObservation::getExpTitle) + .distinct() + .collect(Collectors.toList()); + try { + brAPITrialDAO.getTrialsByName(uniqueTrialNames, program).forEach(existingTrial -> + processAndCacheTrial(existingTrial, program, trialByName) + ); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + return trialByName; + } + + // TODO: also used in other workflow + + /** + * Initializes trials for existing observation units. + * + * @param program The program object. + * @param observationUnitByNameNoScope A map containing observation units by name (without scope). + * @param trialByName A map containing trials by name. (will be modified in place) + * + */ + private void initializeTrialsForExistingObservationUnits(Program program, + Map> observationUnitByNameNoScope, + Map> trialByName) { + if(observationUnitByNameNoScope.size() > 0) { + Set trialDbIds = new HashSet<>(); + Set studyDbIds = new HashSet<>(); + + observationUnitByNameNoScope.values() + .forEach(pio -> { + BrAPIObservationUnit existingOu = pio.getBrAPIObject(); + if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { + throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); + } + + if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { + trialDbIds.add(existingOu.getTrialDbId()); + } else { + studyDbIds.add(existingOu.getStudyDbId()); + } + }); + + //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId + if(!studyDbIds.isEmpty()) { + try { + trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, program)); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + try { + List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); + if (trials.size() != trialDbIds.size()) { + List missingIds = new ArrayList<>(trialDbIds); + missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); + throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + + trials.forEach(trial -> processAndCacheTrial(trial, program, trialByName)); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + } + + /** + * This method processes an existing trial, retrieves the experiment ID from the trial's external references, + * and caches the trial with the corresponding experiment ID in a map. + * + * @param existingTrial The existing BrAPITrial object to be processed and cached. + * @param program The Program object associated with the trial. + * @param trialByNameNoScope The map to cache the trial by its name without program scope. (will be modified in place) + * + * @throws InternalServerException + */ + private void processAndCacheTrial( + BrAPITrial existingTrial, + Program program, + Map> trialByNameNoScope) { + + //get TrialId from existingTrial + BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) + .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); + UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); + + trialByNameNoScope.put( + Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), + new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); + } + + /** + * Fetches trial DbIds for the given study DbIds by using the BrAPI studies API. + * + * @param studyDbIds The set of study DbIds for which to fetch trial DbIds. + * @param program The program associated with the studies. + * @return A set of trial DbIds corresponding to the provided study DbIds. + * @throws ApiException If there was an error while fetching the studies or if a study does not have a trial DbId. + * @throws IllegalStateException If the trial DbId is not set for an existing study. + */ + private Set fetchTrialDbidsForStudies(Set studyDbIds, Program program) throws ApiException { + Set trialDbIds = new HashSet<>(); + List studies = fetchStudiesByDbId(studyDbIds, program); + studies.forEach(study -> { + if (StringUtils.isBlank(study.getTrialDbId())) { + throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); + } + trialDbIds.add(study.getTrialDbId()); + }); + + return trialDbIds; + } + + /** + * Fetches a list of BrAPI studies by their study database IDs for a given program. + * + * This method queries the BrAPIStudyDAO to retrieve studies based on the provided study database IDs and the program. + * It ensures that all requested study database IDs are found in the result set, throwing an IllegalStateException if any are missing. + * + * @param studyDbIds a Set of Strings representing the study database IDs to fetch + * @param program the Program object representing the program context in which to fetch studies + * @return a List of BrAPIStudy objects matching the provided study database IDs + * + * @throws ApiException if there is an issue fetching the studies + * @throws IllegalStateException if any requested study database IDs are not found in the result set + */ + private List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { + List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); + if (studies.size() != studyDbIds.size()) { + List missingIds = new ArrayList<>(studyDbIds); + missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); + throw new IllegalStateException( + "Study not found for studyDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + return studies; + } + + + + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java index fa483a7c4..e57523d58 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java @@ -2,12 +2,12 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ExistingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -public class ProcessStep implements ProcessingStep { +public class ProcessStep implements ProcessingStep { @Override - public ProcessedData process(ExistingData input) { + public ProcessedData process(PendingData input) { // TODO: implement return new ProcessedData(); From 77e4a7e6a475b2ad50e6e2a572346d70c8efb15a Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 29 Apr 2024 17:18:24 -0400 Subject: [PATCH 003/203] Addeed studies pending data to get existing step --- .../steps/GetExistingProcessingStep.java | 153 +++++++++++++++++- 1 file changed, 151 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java index 51997347c..6505d6501 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -3,14 +3,17 @@ import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.server.exceptions.InternalServerException; +import io.reactivex.functions.Function; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPISeason; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; @@ -35,6 +38,7 @@ public class GetExistingProcessingStep implements ProcessingStep> observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); Map> trialByNameNoScope = initializeTrialByNameNoScope(program, observationUnitByNameNoScope, experimentImportRows); - + Map> studyByNameNoScope = initializeStudyByNameNoScope(program, trialByNameNoScope, observationUnitByNameNoScope, experimentImportRows); // TODO: populate rest of data PendingData existing = PendingData.builder() .observationUnitByNameNoScope(observationUnitByNameNoScope) .trialByNameNoScope(trialByNameNoScope) + .studyByNameNoScope(studyByNameNoScope) .build(); return existing; @@ -323,7 +330,149 @@ private List fetchStudiesByDbId(Set studyDbIds, Program prog return studies; } + /** + * Initializes studies by name without scope. + * + * @param program The program object. + * @param trialByNameNoScope A map of trial names with their corresponding pending import objects. + * @param experimentImportRows A list of experiment observation objects. + * @return A map of study names with their corresponding pending import objects. + * @throws InternalServerException If there is an error while processing the method. + */ + private Map> initializeStudyByNameNoScope(Program program, + Map> trialByNameNoScope, + Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> studyByName = new HashMap<>(); + if (trialByNameNoScope.size() != 1) { + return studyByName; + } + + try { + initializeStudiesForExistingObservationUnits(program, studyByName, observationUnitByNameNoScope); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing studies", e); + throw new InternalServerException(e.toString(), e); + } + + List existingStudies; + Optional> trial = getTrialPIO(experimentImportRows, trialByNameNoScope); + + try { + if (trial.isEmpty()) { + // TODO: throw ValidatorException and return 422 + } + UUID experimentId = trial.get().getId(); + existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); + for (BrAPIStudy existingStudy : existingStudies) { + processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); + } + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing studies: ", e); + throw new InternalServerException(e.toString(), e); + } + + return studyByName; + } + + /** + * Retrieves the PendingImportObject of a BrAPITrial based on the given list of ExperimentObservation and trialByNameNoScope map. + * + * @param experimentImportRows The list of ExperimentObservation objects. + * @param trialByNameNoScope The map of trial names to PendingImportObject of BrAPITrial. + * @return The Optional containing the PendingImportObject of BrAPITrial, or an empty Optional if no matching trial is found. + */ + private Optional> getTrialPIO(List experimentImportRows, + Map> trialByNameNoScope) { + Optional expTitle = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID()) && StringUtils.isNotBlank(row.getExpTitle())) + .map(ExperimentObservation::getExpTitle) + .findFirst(); + if (expTitle.isEmpty() && trialByNameNoScope.keySet().stream().findFirst().isEmpty()) { + return Optional.empty(); + } + if(expTitle.isEmpty()) { + expTitle = trialByNameNoScope.keySet().stream().findFirst(); + } + return Optional.ofNullable(trialByNameNoScope.get(expTitle.get())); + } + + + private void initializeStudiesForExistingObservationUnits( + Program program, + Map> studyByName, + Map> observationUnitByNameNoScope + ) throws Exception { + Set studyDbIds = observationUnitByNameNoScope.values() + .stream() + .map(pio -> pio.getBrAPIObject() + .getStudyDbId()) + .collect(Collectors.toSet()); + + List studies = fetchStudiesByDbId(studyDbIds, program); + for (BrAPIStudy study : studies) { + processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); + } + } + + // TODO: used by both workflows + private PendingImportObject processAndCacheStudy( + BrAPIStudy existingStudy, + Program program, + Function getterFunction, + Map> studyMap) throws Exception { + PendingImportObject pendingStudy; + BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); + // map season dbid to year + String seasonDbId = existingStudy.getSeasons().get(0); // It is assumed that the study has only one season + if(StringUtils.isNotBlank(seasonDbId)) { + String seasonYear = seasonDbIdToYear(seasonDbId, program.getId()); + existingStudy.setSeasons(Collections.singletonList(seasonYear)); + } + pendingStudy = new PendingImportObject<>( + ImportObjectState.EXISTING, + (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), + UUID.fromString(xref.getReferenceId()) + ); + studyMap.put( + Utilities.removeProgramKeyAndUnknownAdditionalData(getterFunction.apply(existingStudy), program.getKey()), + pendingStudy + ); + return pendingStudy; + } + + // TODO: used by both workflows + private String seasonDbIdToYear(String seasonDbId, UUID programId) { + String year = null; + // TODO: add season objects to redis cache then just extract year from those + // removing this for now here + //if (this.seasonDbIdToYearCache.containsKey(seasonDbId)) { // get it from cache if possible + // year = this.seasonDbIdToYearCache.get(seasonDbId); + //} else { + year = seasonDbIdToYearFromDatabase(seasonDbId, programId); + // this.seasonDbIdToYearCache.put(seasonDbId, year); + //} + return year; + } + + private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { + BrAPISeason season = null; + try { + season = this.brAPISeasonDAO.getSeasonById(seasonDbId, programId); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + } + Integer yearInt = (season == null) ? null : season.getYear(); + return (yearInt == null) ? "" : yearInt.toString(); + } } From 65cf44d33b469bd8964a573788856b676bd7d52c Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 29 Apr 2024 17:36:59 -0400 Subject: [PATCH 004/203] Added locations pending data to get existing step --- .../steps/GetExistingProcessingStep.java | 57 ++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java index 6505d6501..0dcace200 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -25,6 +25,8 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; @@ -39,6 +41,7 @@ public class GetExistingProcessingStep implements ProcessingStep> observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); Map> trialByNameNoScope = initializeTrialByNameNoScope(program, observationUnitByNameNoScope, experimentImportRows); Map> studyByNameNoScope = initializeStudyByNameNoScope(program, trialByNameNoScope, observationUnitByNameNoScope, experimentImportRows); + // interesting we're using our data model instead of brapi for locations + Map> locationByName = initializeUniqueLocationNames(program, studyByNameNoScope, experimentImportRows); // TODO: populate rest of data PendingData existing = PendingData.builder() .observationUnitByNameNoScope(observationUnitByNameNoScope) .trialByNameNoScope(trialByNameNoScope) .studyByNameNoScope(studyByNameNoScope) + .locationByName(locationByName) .build(); return existing; @@ -475,4 +483,51 @@ private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { return (yearInt == null) ? "" : yearInt.toString(); } + /** + * Initializes unique location names for a program. + * + * @param program The program object. + * @param studyByNameNoScope A map of study names and corresponding BrAPI study objects. + * @param experimentImportRows A list of experiment observation objects for import. + * @return A map of location names and their corresponding pending import objects. + * @throws InternalServerException If there is an error fetching locations. + */ + private Map> initializeUniqueLocationNames(Program program, + Map> studyByNameNoScope, + List experimentImportRows) { + Map> locationByName = new HashMap<>(); + + List existingLocations = new ArrayList<>(); + if(studyByNameNoScope.size() > 0) { + Set locationDbIds = studyByNameNoScope.values() + .stream() + .map(study -> study.getBrAPIObject() + .getLocationDbId()) + .collect(Collectors.toSet()); + try { + existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueLocationNames = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getEnvLocation) + .distinct() + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + try { + existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); + return locationByName; + } + } From 8633a3e389cc205c643f336caaba5b7f707403a1 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 30 Apr 2024 10:37:43 -0400 Subject: [PATCH 005/203] Added dataset pending data to get existing step --- .../steps/GetExistingProcessingStep.java | 78 +++++++++++++++++-- 1 file changed, 70 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java index 0dcace200..ebe0cf5e9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -8,14 +8,11 @@ import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.core.BrAPISeason; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.*; +import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; -import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; -import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; -import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -42,6 +39,7 @@ public class GetExistingProcessingStep implements ProcessingStep> studyByNameNoScope = initializeStudyByNameNoScope(program, trialByNameNoScope, observationUnitByNameNoScope, experimentImportRows); // interesting we're using our data model instead of brapi for locations Map> locationByName = initializeUniqueLocationNames(program, studyByNameNoScope, experimentImportRows); + Map> obsVarDatasetByName = initializeObsVarDatasetByName(program, trialByNameNoScope, experimentImportRows); + // TODO: populate rest of data PendingData existing = PendingData.builder() @@ -78,6 +80,7 @@ public PendingData process(ImportContext input) { .trialByNameNoScope(trialByNameNoScope) .studyByNameNoScope(studyByNameNoScope) .locationByName(locationByName) + .obsVarDatasetByName(obsVarDatasetByName) .build(); return existing; @@ -530,4 +533,63 @@ private Map> initializeUniqueLocati return locationByName; } + /** + * Initializes observation variable dataset by name. + * + * @param program The program associated with the dataset. + * @param trialByNameNoScope The map of trials identified by name without scope. + * @param experimentImportRows The list of experiment observation rows. + * @return The map of observation variable dataset indexed by name. + * + * @throws InternalServerException + */ + private Map> initializeObsVarDatasetByName(Program program, + Map> trialByNameNoScope, + List experimentImportRows) { + Map> obsVarDatasetByName = new HashMap<>(); + + Optional> trialPIO = getTrialPIO(experimentImportRows, trialByNameNoScope); + + if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String datasetId = trialPIO.get().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + try { + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(datasetId)); + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("existing dataset summary not returned from brapi server"); + } + BrAPIListDetails dataSetDetails = brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult(); + processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + return obsVarDatasetByName; + } + + /** + * Process and cache an object of type BrAPIListDetails. + * + * @param existingList The existing list to be processed and cached + * @param obsVarDatasetByName A map of ObsVarDatasets indexed by name (will be modified in place) + * + * @throws IllegalStateException + */ + private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map> obsVarDatasetByName) { + BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for list (dbid): " + existingList.getListDbId())); + obsVarDatasetByName.put(existingList.getListName(), + new PendingImportObject<>(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceId()))); + } + } From ab0364436780552bab8a313f73d875a923962bbe Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 30 Apr 2024 10:56:09 -0400 Subject: [PATCH 006/203] Added germplasm pending data to get existing step --- .../steps/GetExistingProcessingStep.java | 83 ++++++++++++++++++- 1 file changed, 80 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java index ebe0cf5e9..012a17e77 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -10,6 +10,7 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.*; import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.*; @@ -40,6 +41,7 @@ public class GetExistingProcessingStep implements ProcessingStep> locationByName = initializeUniqueLocationNames(program, studyByNameNoScope, experimentImportRows); Map> obsVarDatasetByName = initializeObsVarDatasetByName(program, trialByNameNoScope, experimentImportRows); - - // TODO: populate rest of data + Map> existingGermplasmByGID = initializeExistingGermplasmByGID(program, observationUnitByNameNoScope, experimentImportRows); PendingData existing = PendingData.builder() .observationUnitByNameNoScope(observationUnitByNameNoScope) @@ -81,6 +84,7 @@ public PendingData process(ImportContext input) { .studyByNameNoScope(studyByNameNoScope) .locationByName(locationByName) .obsVarDatasetByName(obsVarDatasetByName) + .existingGermplasmByGID(existingGermplasmByGID) .build(); return existing; @@ -592,4 +596,77 @@ private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceId()))); } + /** + * Initializes existing germplasm objects by germplasm ID (GID). + * + * @param program The program object. + * @param observationUnitByNameNoScope A map of observation unit objects by name. + * @param experimentImportRows A list of experiment observation objects. + * @return A map of existing germplasm objects by germplasm ID. + * + * @throws InternalServerException + */ + private Map> initializeExistingGermplasmByGID(Program program, + Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> existingGermplasmByGID = new HashMap<>(); + + List existingGermplasms = new ArrayList<>(); + if(observationUnitByNameNoScope.size() > 0) { + Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + try { + existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueGermplasmGIDs = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getGid) + .distinct() + .collect(Collectors.toList()); + + try { + existingGermplasms.addAll(getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + existingGermplasms.forEach(existingGermplasm -> { + BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); + }); + return existingGermplasmByGID; + } + + /** + * Retrieves a list of germplasm with the specified accession numbers. + * + * @param germplasmAccessionNumbers The list of accession numbers to search for. + * @param programId The ID of the program. + * @return An ArrayList of BrAPIGermplasm objects that match the accession numbers. + * @throws ApiException if there is an error retrieving the germplasm. + */ + private ArrayList getGermplasmByAccessionNumber( + List germplasmAccessionNumbers, + UUID programId) throws ApiException { + List germplasmList = brAPIGermplasmDAO.getGermplasm(programId); + ArrayList resultGermplasm = new ArrayList<>(); + // Search for accession number matches + for (BrAPIGermplasm germplasm : germplasmList) { + for (String accessionNumber : germplasmAccessionNumbers) { + if (germplasm.getAccessionNumber() + .equals(accessionNumber)) { + resultGermplasm.add(germplasm); + break; + } + } + } + return resultGermplasm; + } + } From 9d2e108bbce6beeec72b4571fb5eff96ce750f11 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 1 May 2024 11:31:14 -0400 Subject: [PATCH 007/203] create Middleware class --- .../AppendOverwritePhenotypesWorkflow.java | 17 +++- .../middleware/ExpUnit/ExpUnitMiddleware.java | 8 ++ .../ExpUnit/GetExistingBrAPIData.java | 87 +++++++++++++++++++ .../ExpUnit/ValidateAllRowsHaveIDs.java | 38 ++++++++ .../experiment/middleware/Middleware.java | 34 ++++++++ .../model/ExpImportProcessErrorConstants.java | 24 +++++ .../model/ExpUnitMiddlewareContext.java | 31 +++++++ 7 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ExpUnitMiddleware.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessErrorConstants.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 970fc25f3..39e629d36 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -1,6 +1,10 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite; import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit.GetExistingBrAPIData; +import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit.ValidateAllRowsHaveIDs; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.Workflow; @@ -8,10 +12,19 @@ @Prototype public class AppendOverwritePhenotypesWorkflow implements Workflow { + ExpUnitMiddleware middleware; + GetExistingBrAPIData getExistingBrAPIData; + public AppendOverwritePhenotypesWorkflow(GetExistingBrAPIData getExistingBrAPIData) { + this.middleware.link( + new ValidateAllRowsHaveIDs(), + getExistingBrAPIData + ); + } @Override public ProcessedData process(ImportContext context) { - - // validate that all rows have an ObsUnitID + ExpUnitMiddlewareContext workflowContext = new ExpUnitMiddlewareContext(); + workflowContext.setImportContext(context); + this.middleware.process(workflowContext); // TODO: implement return new ProcessedData(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ExpUnitMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ExpUnitMiddleware.java new file mode 100644 index 000000000..2cfe6c4a6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ExpUnitMiddleware.java @@ -0,0 +1,8 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit; + +import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.Middleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +public abstract class ExpUnitMiddleware extends Middleware { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java new file mode 100644 index 000000000..80c458903 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java @@ -0,0 +1,87 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessErrorConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class GetExistingBrAPIData extends ExpUnitMiddleware { + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public GetExistingBrAPIData(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + return processNext(context); + } + + private Map> fetchReferenceObservationUnits( + Set referenceOUIds, + Program program + ) throws ApiException { + Map> pendingUnitById = new HashMap<>(); + try { + // Retrieve reference Observation Units based on IDs + List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( + new ArrayList(referenceOUIds), + program + ); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + if (referenceObsUnits.size() == referenceOUIds.size()) { + // Iterate through reference Observation Units + referenceObsUnits.forEach(unit -> { + // Get external reference for the Observation Unit + BrAPIExternalReference unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) + .orElseThrow(() -> new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID")); + + // Set pending Observation Unit by its ID + pendingUnitById.put( + unitXref.getReferenceId(), + new PendingImportObject( + ImportObjectState.EXISTING, unit, UUID.fromString(unitXref.getReferenceId())) + ); + }); + } else { + // Handle missing Observation Unit IDs + List missingIds = new ArrayList<>(referenceOUIds); + Set fetchedIds = referenceObsUnits.stream().map(unit -> + Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) + .orElseThrow(() -> new InternalServerException("External reference does not exist for Deltabreed ObservationUnit ID")) + .getReferenceId()) + .collect(Collectors.toSet()); + missingIds.removeAll(fetchedIds); + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessErrorConstants.COMMA_DELIMITER, missingIds)); + } + + return pendingUnitById; + } catch (ApiException e) { + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new ApiException(e); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java new file mode 100644 index 000000000..26c9f3466 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java @@ -0,0 +1,38 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit; + +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +import java.util.HashSet; +import java.util.Set; + +public class ValidateAllRowsHaveIDs extends ExpUnitMiddleware { + private boolean hasAllReferenceUnitIds = true; + private boolean hasNoReferenceUnitIds = true; + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + context.setReferenceOUIds(collateReferenceOUIds(context)); + return processNext(context); + } + private Set collateReferenceOUIds(ExpUnitMiddlewareContext context) { + Set referenceOUIds = new HashSet<>(); + for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + + // Check if ObsUnitID is blank + if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { + hasAllReferenceUnitIds = false; + } else if (referenceOUIds.contains(importRow.getObsUnitID())) { + // Throw exception if ObsUnitID is repeated + throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); + } else { + // Add ObsUnitID to referenceOUIds + referenceOUIds.add(importRow.getObsUnitID()); + hasNoReferenceUnitIds = false; + } + } + return referenceOUIds; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java new file mode 100644 index 000000000..f9dfb31e6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java @@ -0,0 +1,34 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.middleware; + +public abstract class Middleware { + + Middleware next; + + /** + * Builds chains of middleware objects. + */ + public static Middleware link(Middleware first, Middleware... chain) { + Middleware head = first; + for (Middleware nextInChain: chain) { + head.next = nextInChain; + head = nextInChain; + } + return first; + } + + /** + * Subclasses will implement this method with processing steps. + */ + public abstract boolean process(T context); + + /** + * Runs check on the next object in chain or ends traversing if we're in + * last object in chain. + */ + protected boolean processNext(T context) { + if (next == null) { + return true; + } + return next.process(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessErrorConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessErrorConstants.java new file mode 100644 index 000000000..8509d74f6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessErrorConstants.java @@ -0,0 +1,24 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import com.fasterxml.jackson.annotation.JsonValue; + +public class ExpImportProcessErrorConstants { + + public static final CharSequence COMMA_DELIMITER = ","; + public enum ExpImportProcessErrMessage { + + MISSING_OBS_UNIT_ID_ERROR("Experimental entities are missing ObsUnitIDs"), + PREEXISTING_EXPERIMENT_TITLE("Experiment Title already exists"); + private String value; + + ExpImportProcessErrMessage(String value) { + this.value = value; + } + + @JsonValue + public String getValue() { + return value; + } + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java new file mode 100644 index 000000000..2ce05216b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java @@ -0,0 +1,31 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import lombok.*; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.ProgramLocation; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class ExpUnitMiddlewareContext { + @Getter + @Setter + private ImportContext importContext; + @Getter + @Setter + private Set referenceOUIds = new HashSet<>(); + private Map> pendingTrialByOUId = new HashMap<>(); + private Map> pendingStudyByOUId = new HashMap<>(); + private Map> pendingObsUnitByOUId = new HashMap<>(); + private Map> pendingObsDatasetByOUId = new HashMap<>(); + private Map> pendingLocationByOUId = new HashMap<>(); + private Map> pendingGermplasmByOUId = new HashMap<>(); + +} From d97f27c41cc59e6d6cf2c09db97c8630a6430f1f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 1 May 2024 20:23:04 -0400 Subject: [PATCH 008/203] add comensation methods to Middleware class --- .../experiment/middleware/Middleware.java | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java index f9dfb31e6..d0f9f06c2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java @@ -3,6 +3,7 @@ public abstract class Middleware { Middleware next; + Middleware prior; /** * Builds chains of middleware objects. @@ -10,6 +11,7 @@ public abstract class Middleware { public static Middleware link(Middleware first, Middleware... chain) { Middleware head = first; for (Middleware nextInChain: chain) { + nextInChain.prior = head; head.next = nextInChain; head = nextInChain; } @@ -17,13 +19,16 @@ public static Middleware link(Middleware first, Middleware... chain) { } /** - * Subclasses will implement this method with processing steps. + * Subclasses will implement this local transaction. */ public abstract boolean process(T context); - /** - * Runs check on the next object in chain or ends traversing if we're in - * last object in chain. + * Subclasses will implement this method to handle errors and possibly undo the local transaction. + */ + public abstract boolean compensate(T context); + /** + * Processes the next local transaction or ends traversing if we're at the + * last local transaction of the transaction. */ protected boolean processNext(T context) { if (next == null) { @@ -31,4 +36,15 @@ protected boolean processNext(T context) { } return next.process(context); } + + /** + * Runs the compensating local transaction for the prior local transaction or ends traversing if + * we're at the first local transaction of the transaction. + */ + protected boolean compensatePrior(T context) { + if (prior == null) { + return true; + } + return prior.compensate(context); + } } From 3a0481fea4c43422bfd750b5ca587b681c1b4bac Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 2 May 2024 09:36:28 -0400 Subject: [PATCH 009/203] add compensate method to middleware --- .../ExpUnit/GetExistingBrAPIData.java | 77 +++++++++++++------ .../ExpUnit/ValidateAllRowsHaveIDs.java | 17 +++- .../experiment/middleware/Middleware.java | 8 +- .../experiment/model/MiddlewareError.java | 21 +++++ 4 files changed, 97 insertions(+), 26 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java index 80c458903..01ae96ef5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java @@ -14,6 +14,7 @@ import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessErrorConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -34,54 +35,86 @@ public GetExistingBrAPIData(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { @Override public boolean process(ExpUnitMiddlewareContext context) { + return processNext(context); } + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // tag an error if it occurred in this local transaction + error.tag(this.getClass().getName()); + + // handle the error in the prior local transaction + return compensatePrior(context, error); + } private Map> fetchReferenceObservationUnits( - Set referenceOUIds, - Program program - ) throws ApiException { + ExpUnitMiddlewareContext context) { Map> pendingUnitById = new HashMap<>(); try { // Retrieve reference Observation Units based on IDs List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( - new ArrayList(referenceOUIds), - program + new ArrayList(context.getReferenceOUIds()), + context.getImportContext().getProgram() ); // Construct the DeltaBreed observation unit source for external references String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - if (referenceObsUnits.size() == referenceOUIds.size()) { + if (referenceObsUnits.size() == context.getReferenceOUIds().size()) { + // Iterate through reference Observation Units - referenceObsUnits.forEach(unit -> { - // Get external reference for the Observation Unit - BrAPIExternalReference unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) - .orElseThrow(() -> new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID")); - - // Set pending Observation Unit by its ID - pendingUnitById.put( - unitXref.getReferenceId(), - new PendingImportObject( - ImportObjectState.EXISTING, unit, UUID.fromString(unitXref.getReferenceId())) + for (BrAPIObservationUnit unit : referenceObsUnits) {// Get external reference for the Observation Unit + Optional unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource); + unitXref.ifPresentOrElse( + xref -> { + + // Set pending Observation Unit by its ID + pendingUnitById.put( + xref.getReferenceId(), + new PendingImportObject<>( + ImportObjectState.EXISTING, unit, UUID.fromString(xref.getReferenceId())) + ); + }, + () -> { + + // but throw an error if no unit ID + this.compensate(context, new MiddlewareError(() -> { + throw new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID"); + })); + } ); - }); + + + } } else { - // Handle missing Observation Unit IDs - List missingIds = new ArrayList<>(referenceOUIds); + // Handle case of missing Observation Units in data store + List missingIds = new ArrayList<>(context.getReferenceOUIds()); Set fetchedIds = referenceObsUnits.stream().map(unit -> Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) .orElseThrow(() -> new InternalServerException("External reference does not exist for Deltabreed ObservationUnit ID")) .getReferenceId()) .collect(Collectors.toSet()); missingIds.removeAll(fetchedIds); - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessErrorConstants.COMMA_DELIMITER, missingIds)); + + // throw error reporting any reference IDs with no corresponding stored unit in the brapi data store + this.compensate(context, new MiddlewareError(() -> { + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessErrorConstants.COMMA_DELIMITER, missingIds)); + })); } return pendingUnitById; } catch (ApiException e) { - log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new ApiException(e); + + // throw an error if problem getting data from the brapi data store + this.compensate(context, new MiddlewareError(() -> { + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); + try { + throw new ApiException(e); + } catch (ApiException ex) { + throw new RuntimeException(ex); + } + })); } + return pendingUnitById; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java index 26c9f3466..32219a821 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java @@ -3,6 +3,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import java.util.HashSet; import java.util.Set; @@ -16,6 +17,16 @@ public boolean process(ExpUnitMiddlewareContext context) { context.setReferenceOUIds(collateReferenceOUIds(context)); return processNext(context); } + + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // tag an error if it occurred in this local transaction + error.tag(this.getClass().getName()); + + // undo the prior local transaction + return compensatePrior(context, error); + } + private Set collateReferenceOUIds(ExpUnitMiddlewareContext context) { Set referenceOUIds = new HashSet<>(); for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { @@ -25,8 +36,12 @@ private Set collateReferenceOUIds(ExpUnitMiddlewareContext context) { if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { hasAllReferenceUnitIds = false; } else if (referenceOUIds.contains(importRow.getObsUnitID())) { + // Throw exception if ObsUnitID is repeated - throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); + this.compensate(context, new MiddlewareError(()->{ + throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); + })); + } else { // Add ObsUnitID to referenceOUIds referenceOUIds.add(importRow.getObsUnitID()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java index d0f9f06c2..d2d7443b8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java @@ -1,5 +1,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.middleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; + public abstract class Middleware { Middleware next; @@ -25,7 +27,7 @@ public static Middleware link(Middleware first, Middleware... chain) { /** * Subclasses will implement this method to handle errors and possibly undo the local transaction. */ - public abstract boolean compensate(T context); + public abstract boolean compensate(T context, MiddlewareError error); /** * Processes the next local transaction or ends traversing if we're at the * last local transaction of the transaction. @@ -41,10 +43,10 @@ protected boolean processNext(T context) { * Runs the compensating local transaction for the prior local transaction or ends traversing if * we're at the first local transaction of the transaction. */ - protected boolean compensatePrior(T context) { + protected boolean compensatePrior(T context, MiddlewareError error) { if (prior == null) { return true; } - return prior.compensate(context); + return prior.compensate(context, error); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java new file mode 100644 index 000000000..49960ae3a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java @@ -0,0 +1,21 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import lombok.Getter; +import lombok.Setter; + +public class MiddlewareError { + @Getter + @Setter + String localTransactionName; + Runnable handler; + + public MiddlewareError(Runnable handler) { + this.handler = handler; + } + + public void tag(String name) { + if (this.getLocalTransactionName() == null) { + this.setLocalTransactionName(name); + } + } +} From 2741485c7911eadf2811f6ef439fef0bbe913859 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 2 May 2024 13:33:07 -0400 Subject: [PATCH 010/203] Work on breaking out common workflow code --- .../experiment/ExperimentUtilities.java | 2 + .../steps/GetExistingProcessingStep.java | 231 +----------------- .../experiment/services/StudyService.java | 120 +++++++++ .../experiment/services/TrialService.java | 182 ++++++++++++++ 4 files changed, 315 insertions(+), 220 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index bd90ee222..e0bdfe432 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -9,6 +9,7 @@ public class ExperimentUtilities { public static final CharSequence COMMA_DELIMITER = ","; + public static final String TIMESTAMP_PREFIX = "TS:"; public static List importRowsToExperimentObservations(List importRows) { return importRows.stream() @@ -17,4 +18,5 @@ public static List importRowsToExperimentObservations(Lis } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java index 012a17e77..d01324e4f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -22,6 +22,8 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.StudyService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.TrialService; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.services.ProgramLocationService; @@ -38,10 +40,11 @@ public class GetExistingProcessingStep implements ProcessingStep> initializeTrialByNameNoScope(Program program, Map> observationUnitByNameNoScope, - List experimentImportRows) { - Map> trialByName = new HashMap<>(); - - initializeTrialsForExistingObservationUnits(program, observationUnitByNameNoScope, trialByName); - - List uniqueTrialNames = experimentImportRows.stream() - .filter(row -> StringUtils.isBlank(row.getObsUnitID())) - .map(ExperimentObservation::getExpTitle) - .distinct() - .collect(Collectors.toList()); - try { - brAPITrialDAO.getTrialsByName(uniqueTrialNames, program).forEach(existingTrial -> - processAndCacheTrial(existingTrial, program, trialByName) - ); - } catch (ApiException e) { - log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - - return trialByName; - } - - // TODO: also used in other workflow - - /** - * Initializes trials for existing observation units. - * - * @param program The program object. - * @param observationUnitByNameNoScope A map containing observation units by name (without scope). - * @param trialByName A map containing trials by name. (will be modified in place) - * - */ - private void initializeTrialsForExistingObservationUnits(Program program, - Map> observationUnitByNameNoScope, - Map> trialByName) { - if(observationUnitByNameNoScope.size() > 0) { - Set trialDbIds = new HashSet<>(); - Set studyDbIds = new HashSet<>(); - - observationUnitByNameNoScope.values() - .forEach(pio -> { - BrAPIObservationUnit existingOu = pio.getBrAPIObject(); - if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { - throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); - } - - if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { - trialDbIds.add(existingOu.getTrialDbId()); - } else { - studyDbIds.add(existingOu.getStudyDbId()); - } - }); - - //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId - if(!studyDbIds.isEmpty()) { - try { - trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, program)); - } catch (ApiException e) { - log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - try { - List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); - if (trials.size() != trialDbIds.size()) { - List missingIds = new ArrayList<>(trialDbIds); - missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); - throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); - } - - trials.forEach(trial -> processAndCacheTrial(trial, program, trialByName)); - } catch (ApiException e) { - log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - } - - /** - * This method processes an existing trial, retrieves the experiment ID from the trial's external references, - * and caches the trial with the corresponding experiment ID in a map. - * - * @param existingTrial The existing BrAPITrial object to be processed and cached. - * @param program The Program object associated with the trial. - * @param trialByNameNoScope The map to cache the trial by its name without program scope. (will be modified in place) - * - * @throws InternalServerException - */ - private void processAndCacheTrial( - BrAPITrial existingTrial, - Program program, - Map> trialByNameNoScope) { - - //get TrialId from existingTrial - BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), - String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) - .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); - UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); - - trialByNameNoScope.put( - Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), - new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); - } - - /** - * Fetches trial DbIds for the given study DbIds by using the BrAPI studies API. - * - * @param studyDbIds The set of study DbIds for which to fetch trial DbIds. - * @param program The program associated with the studies. - * @return A set of trial DbIds corresponding to the provided study DbIds. - * @throws ApiException If there was an error while fetching the studies or if a study does not have a trial DbId. - * @throws IllegalStateException If the trial DbId is not set for an existing study. - */ - private Set fetchTrialDbidsForStudies(Set studyDbIds, Program program) throws ApiException { - Set trialDbIds = new HashSet<>(); - List studies = fetchStudiesByDbId(studyDbIds, program); - studies.forEach(study -> { - if (StringUtils.isBlank(study.getTrialDbId())) { - throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); - } - trialDbIds.add(study.getTrialDbId()); - }); - - return trialDbIds; - } - - /** - * Fetches a list of BrAPI studies by their study database IDs for a given program. - * - * This method queries the BrAPIStudyDAO to retrieve studies based on the provided study database IDs and the program. - * It ensures that all requested study database IDs are found in the result set, throwing an IllegalStateException if any are missing. - * - * @param studyDbIds a Set of Strings representing the study database IDs to fetch - * @param program the Program object representing the program context in which to fetch studies - * @return a List of BrAPIStudy objects matching the provided study database IDs - * - * @throws ApiException if there is an issue fetching the studies - * @throws IllegalStateException if any requested study database IDs are not found in the result set - */ - private List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { - List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); - if (studies.size() != studyDbIds.size()) { - List missingIds = new ArrayList<>(studyDbIds); - missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); - throw new IllegalStateException( - "Study not found for studyDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); - } - return studies; - } - /** * Initializes studies by name without scope. * @@ -383,7 +226,7 @@ private Map> initializeStudyByNameNoScop UUID experimentId = trial.get().getId(); existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); for (BrAPIStudy existingStudy : existingStudies) { - processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); + studyService.processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); } } catch (ApiException e) { log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); @@ -434,60 +277,8 @@ private void initializeStudiesForExistingObservationUnits( List studies = fetchStudiesByDbId(studyDbIds, program); for (BrAPIStudy study : studies) { - processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); - } - } - - // TODO: used by both workflows - private PendingImportObject processAndCacheStudy( - BrAPIStudy existingStudy, - Program program, - Function getterFunction, - Map> studyMap) throws Exception { - PendingImportObject pendingStudy; - BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) - .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); - // map season dbid to year - String seasonDbId = existingStudy.getSeasons().get(0); // It is assumed that the study has only one season - if(StringUtils.isNotBlank(seasonDbId)) { - String seasonYear = seasonDbIdToYear(seasonDbId, program.getId()); - existingStudy.setSeasons(Collections.singletonList(seasonYear)); - } - pendingStudy = new PendingImportObject<>( - ImportObjectState.EXISTING, - (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), - UUID.fromString(xref.getReferenceId()) - ); - studyMap.put( - Utilities.removeProgramKeyAndUnknownAdditionalData(getterFunction.apply(existingStudy), program.getKey()), - pendingStudy - ); - return pendingStudy; - } - - // TODO: used by both workflows - private String seasonDbIdToYear(String seasonDbId, UUID programId) { - String year = null; - // TODO: add season objects to redis cache then just extract year from those - // removing this for now here - //if (this.seasonDbIdToYearCache.containsKey(seasonDbId)) { // get it from cache if possible - // year = this.seasonDbIdToYearCache.get(seasonDbId); - //} else { - year = seasonDbIdToYearFromDatabase(seasonDbId, programId); - // this.seasonDbIdToYearCache.put(seasonDbId, year); - //} - return year; - } - - private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { - BrAPISeason season = null; - try { - season = this.brAPISeasonDAO.getSeasonById(seasonDbId, programId); - } catch (ApiException e) { - log.error(Utilities.generateApiExceptionLogMessage(e), e); + studyService.processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); } - Integer yearInt = (season == null) ? null : season.getYear(); - return (yearInt == null) ? "" : yearInt.toString(); } /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java new file mode 100644 index 000000000..daca5630f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java @@ -0,0 +1,120 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Prototype; +import io.reactivex.functions.Function; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPISeason; +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +@Singleton +@Slf4j +public class StudyService { + + private final BrAPISeasonDAO brAPISeasonDAO; + private final BrAPIStudyDAO brAPIStudyDAO; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public StudyService(BrAPISeasonDAO brAPISeasonDAO, + BrAPIStudyDAO brAPIStudyDAO) { + this.brAPISeasonDAO = brAPISeasonDAO; + this.brAPIStudyDAO = brAPIStudyDAO; + } + + // TODO: used by both workflows + public PendingImportObject processAndCacheStudy( + BrAPIStudy existingStudy, + Program program, + Function getterFunction, + Map> studyMap) throws Exception { + PendingImportObject pendingStudy; + BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); + // map season dbid to year + String seasonDbId = existingStudy.getSeasons().get(0); // It is assumed that the study has only one season + if(StringUtils.isNotBlank(seasonDbId)) { + String seasonYear = seasonDbIdToYear(seasonDbId, program.getId()); + existingStudy.setSeasons(Collections.singletonList(seasonYear)); + } + pendingStudy = new PendingImportObject<>( + ImportObjectState.EXISTING, + (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), + UUID.fromString(xref.getReferenceId()) + ); + studyMap.put( + Utilities.removeProgramKeyAndUnknownAdditionalData(getterFunction.apply(existingStudy), program.getKey()), + pendingStudy + ); + return pendingStudy; + } + + // TODO: used by both workflows + private String seasonDbIdToYear(String seasonDbId, UUID programId) { + String year = null; + // TODO: add season objects to redis cache then just extract year from those + // removing this for now here + //if (this.seasonDbIdToYearCache.containsKey(seasonDbId)) { // get it from cache if possible + // year = this.seasonDbIdToYearCache.get(seasonDbId); + //} else { + year = seasonDbIdToYearFromDatabase(seasonDbId, programId); + // this.seasonDbIdToYearCache.put(seasonDbId, year); + //} + return year; + } + + // TODO: used by both workflows + private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { + BrAPISeason season = null; + try { + season = this.brAPISeasonDAO.getSeasonById(seasonDbId, programId); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + } + Integer yearInt = (season == null) ? null : season.getYear(); + return (yearInt == null) ? "" : yearInt.toString(); + } + + /** + * Fetches a list of BrAPI studies by their study database IDs for a given program. + * + * This method queries the BrAPIStudyDAO to retrieve studies based on the provided study database IDs and the program. + * It ensures that all requested study database IDs are found in the result set, throwing an IllegalStateException if any are missing. + * + * @param studyDbIds a Set of Strings representing the study database IDs to fetch + * @param program the Program object representing the program context in which to fetch studies + * @return a List of BrAPIStudy objects matching the provided study database IDs + * + * @throws ApiException if there is an issue fetching the studies + * @throws IllegalStateException if any requested study database IDs are not found in the result set + */ + private List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { + List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); + if (studies.size() != studyDbIds.size()) { + List missingIds = new ArrayList<>(studyDbIds); + missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); + throw new IllegalStateException( + "Study not found for studyDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + return studies; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java new file mode 100644 index 000000000..76e365cca --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java @@ -0,0 +1,182 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +@Singleton +@Slf4j +public class TrialService { + private final BrAPITrialDAO brAPITrialDAO; + + private final StudyService studyService; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public TrialService(BrAPITrialDAO brAPITrialDAO, + StudyService studyService) { + this.brAPITrialDAO = brAPITrialDAO; + this.studyService = studyService; + } + + // TODO: also used in other workflow + + /** + * Initializes trials for existing observation units. + * + * @param program The program object. + * @param observationUnitByNameNoScope A map containing observation units by name (without scope). + * @param trialByName A map containing trials by name. (will be modified in place) + * + */ + public void initializeTrialsForExistingObservationUnits(Program program, + Map> observationUnitByNameNoScope, + Map> trialByName) { + if(observationUnitByNameNoScope.size() > 0) { + Set trialDbIds = new HashSet<>(); + Set studyDbIds = new HashSet<>(); + + observationUnitByNameNoScope.values() + .forEach(pio -> { + BrAPIObservationUnit existingOu = pio.getBrAPIObject(); + if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { + throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); + } + + if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { + trialDbIds.add(existingOu.getTrialDbId()); + } else { + studyDbIds.add(existingOu.getStudyDbId()); + } + }); + + //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId + if(!studyDbIds.isEmpty()) { + try { + trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, program)); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + try { + List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); + if (trials.size() != trialDbIds.size()) { + List missingIds = new ArrayList<>(trialDbIds); + missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); + throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + + trials.forEach(trial -> processAndCacheTrial(trial, program, trialByName)); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + } + + /** + * Fetches trial DbIds for the given study DbIds by using the BrAPI studies API. + * + * @param studyDbIds The set of study DbIds for which to fetch trial DbIds. + * @param program The program associated with the studies. + * @return A set of trial DbIds corresponding to the provided study DbIds. + * @throws ApiException If there was an error while fetching the studies or if a study does not have a trial DbId. + * @throws IllegalStateException If the trial DbId is not set for an existing study. + */ + private Set fetchTrialDbidsForStudies(Set studyDbIds, Program program) throws ApiException { + Set trialDbIds = new HashSet<>(); + List studies = fetchStudiesByDbId(studyDbIds, program); + studies.forEach(study -> { + if (StringUtils.isBlank(study.getTrialDbId())) { + throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); + } + trialDbIds.add(study.getTrialDbId()); + }); + + return trialDbIds; + } + + /** + * This method processes an existing trial, retrieves the experiment ID from the trial's external references, + * and caches the trial with the corresponding experiment ID in a map. + * + * @param existingTrial The existing BrAPITrial object to be processed and cached. + * @param program The Program object associated with the trial. + * @param trialByNameNoScope The map to cache the trial by its name without program scope. (will be modified in place) + * + * @throws InternalServerException + */ + private void processAndCacheTrial( + BrAPITrial existingTrial, + Program program, + Map> trialByNameNoScope) { + + //get TrialId from existingTrial + BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) + .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); + UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); + + trialByNameNoScope.put( + Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), + new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); + } + + /** + * Initializes trials by name without scope for the given program. + * + * @param program the program to initialize trials for + * @param observationUnitByNameNoScope a map of observation units by name without scope + * @param experimentImportRows a list of experiment observation rows + * @return a map of trials by name with pending import objects + * + * @throws InternalServerException + */ + private Map> initializeTrialByNameNoScope(Program program, Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> trialByName = new HashMap<>(); + + initializeTrialsForExistingObservationUnits(program, observationUnitByNameNoScope, trialByName); + + List uniqueTrialNames = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID())) + .map(ExperimentObservation::getExpTitle) + .distinct() + .collect(Collectors.toList()); + try { + brAPITrialDAO.getTrialsByName(uniqueTrialNames, program).forEach(existingTrial -> + processAndCacheTrial(existingTrial, program, trialByName) + ); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + return trialByName; + } + +} From 8f761aea7d15bc4ef2d150a93b3c818a59532ecb Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 2 May 2024 14:48:32 -0400 Subject: [PATCH 011/203] create ExpUnitContextService --- .../AppendOverwritePhenotypesWorkflow.java | 17 ++-- .../ExpUnitContextService.java | 96 +++++++++++++++++++ .../middleware}/ExpUnitMiddleware.java | 2 +- .../middleware}/GetExistingBrAPIData.java | 12 +-- .../middleware}/ValidateAllRowsHaveIDs.java | 4 +- .../appendoverwrite/model/ExpUnitContext.java | 29 ++++++ .../model/ExpUnitMiddlewareContext.java | 29 +----- 7 files changed, 149 insertions(+), 40 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{middleware/ExpUnit => appendoverwrite/middleware}/ExpUnitMiddleware.java (90%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{middleware/ExpUnit => appendoverwrite/middleware}/GetExistingBrAPIData.java (93%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{middleware/ExpUnit => appendoverwrite/middleware}/ValidateAllRowsHaveIDs.java (93%) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 39e629d36..fe2e1ae3b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -1,23 +1,28 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite; import io.micronaut.context.annotation.Prototype; -import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit.GetExistingBrAPIData; -import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit.ValidateAllRowsHaveIDs; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.GetExistingBrAPIData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ValidateAllRowsHaveIDs; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.Workflow; +import javax.inject.Inject; +import javax.inject.Provider; + + @Prototype public class AppendOverwritePhenotypesWorkflow implements Workflow { ExpUnitMiddleware middleware; - GetExistingBrAPIData getExistingBrAPIData; - public AppendOverwritePhenotypesWorkflow(GetExistingBrAPIData getExistingBrAPIData) { + Provider getExistingBrAPIDataProvider; + @Inject + public AppendOverwritePhenotypesWorkflow(Provider getExistingBrAPIDataProvider) { this.middleware.link( new ValidateAllRowsHaveIDs(), - getExistingBrAPIData + getExistingBrAPIDataProvider.get() ); } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java new file mode 100644 index 000000000..39a5e0215 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java @@ -0,0 +1,96 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessErrorConstants; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class ExpUnitContextService { + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public ExpUnitContextService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + } + + /** + * Retrieves reference Observation Units based on a set of reference Observation Unit IDs and a Program. + * Constructs DeltaBreed observation unit source for external references and sets up pending Observation Units. + * + * @param referenceOUIds A set of reference Observation Unit IDs to retrieve + * @param program The Program associated with the Observation Units + * @return A Map containing pending Observation Units by their ID + * @throws ApiException if an error occurs during the process + */ + public Map> fetchReferenceObservationUnits( + Set referenceOUIds, + Program program + ) throws ApiException { + Map> pendingUnitById = new HashMap<>(); + try { + // Retrieve reference Observation Units based on IDs + List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( + new ArrayList(referenceOUIds), + program + ); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + if (referenceObsUnits.size() == referenceOUIds.size()) { + for (BrAPIObservationUnit unit : referenceObsUnits) {// Iterate through reference Observation Units + + // Get external reference for the Observation Unit + Optional unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource); + unitXref.ifPresentOrElse( + xref -> { + + // Set pending Observation Unit by its ID + pendingUnitById.put( + xref.getReferenceId(), + new PendingImportObject<>( + ImportObjectState.EXISTING, unit, UUID.fromString(xref.getReferenceId())) + ); + }, + () -> { + + // but throw an error if no unit ID + throw new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID"); + } + ); + } + } else {// Handle case of missing Observation Units in data store + List missingIds = new ArrayList<>(referenceOUIds); + Set fetchedIds = referenceObsUnits.stream() + .filter(unit ->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).isPresent()) + .map(unit->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).get().getReferenceId()) + .collect(Collectors.toSet()); + missingIds.removeAll(fetchedIds); + + // throw error reporting any reference IDs with no corresponding stored unit in the brapi data store + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessErrorConstants.COMMA_DELIMITER, missingIds)); + } + + return pendingUnitById; + } catch (ApiException e) { + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new ApiException(e); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ExpUnitMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java similarity index 90% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ExpUnitMiddleware.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java index 2cfe6c4a6..81e4e1a0d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ExpUnitMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.Middleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java similarity index 93% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java index 01ae96ef5..e87583d6b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; @@ -7,15 +7,13 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessErrorConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; -import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; @@ -53,14 +51,14 @@ private Map> fetchReferenceObs try { // Retrieve reference Observation Units based on IDs List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( - new ArrayList(context.getReferenceOUIds()), + new ArrayList(context.getExpUnitContext().getReferenceOUIds()), context.getImportContext().getProgram() ); // Construct the DeltaBreed observation unit source for external references String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - if (referenceObsUnits.size() == context.getReferenceOUIds().size()) { + if (referenceObsUnits.size() == context.getExpUnitContext().getReferenceOUIds().size()) { // Iterate through reference Observation Units for (BrAPIObservationUnit unit : referenceObsUnits) {// Get external reference for the Observation Unit @@ -88,7 +86,7 @@ private Map> fetchReferenceObs } } else { // Handle case of missing Observation Units in data store - List missingIds = new ArrayList<>(context.getReferenceOUIds()); + List missingIds = new ArrayList<>(context.getExpUnitContext().getReferenceOUIds()); Set fetchedIds = referenceObsUnits.stream().map(unit -> Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) .orElseThrow(() -> new InternalServerException("External reference does not exist for Deltabreed ObservationUnit ID")) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java similarity index 93% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java index 32219a821..45c728453 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java @@ -1,7 +1,7 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java new file mode 100644 index 000000000..50ca1f4d2 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java @@ -0,0 +1,29 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; + +import lombok.Getter; +import lombok.Setter; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.ProgramLocation; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@Getter +@Setter +public class ExpUnitContext { + private Set referenceOUIds = new HashSet<>(); + private Map> pendingTrialByOUId = new HashMap<>(); + private Map> pendingStudyByOUId = new HashMap<>(); + private Map> pendingObsUnitByOUId = new HashMap<>(); + private Map> pendingObsDatasetByOUId = new HashMap<>(); + private Map> pendingLocationByOUId = new HashMap<>(); + private Map> pendingGermplasmByOUId = new HashMap<>(); + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java index 2ce05216b..6a8f9183e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java @@ -1,31 +1,12 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.model; import lombok.*; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.model.ProgramLocation; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +@Getter +@Setter public class ExpUnitMiddlewareContext { - @Getter - @Setter - private ImportContext importContext; - @Getter - @Setter - private Set referenceOUIds = new HashSet<>(); - private Map> pendingTrialByOUId = new HashMap<>(); - private Map> pendingStudyByOUId = new HashMap<>(); - private Map> pendingObsUnitByOUId = new HashMap<>(); - private Map> pendingObsDatasetByOUId = new HashMap<>(); - private Map> pendingLocationByOUId = new HashMap<>(); - private Map> pendingGermplasmByOUId = new HashMap<>(); + private ImportContext importContext; + private ExpUnitContext expUnitContext; } From 8fe5fe29d4379eebf2aa649d90ab91948bfef160 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 2 May 2024 16:04:19 -0400 Subject: [PATCH 012/203] add mapPendingOUsByName service method --- .../experiment/ExperimentUtilities.java | 6 +++ .../ExpUnitContextService.java | 44 ++++++++++++------- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index e0bdfe432..92d333ce9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -17,6 +17,12 @@ public static List importRowsToExperimentObservations(Lis .collect(Collectors.toList()); } + public static String createObservationUnitKey(ExperimentObservation importRow) { + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); + } + public static String createObservationUnitKey(String studyName, String obsUnitName) { + return studyName + obsUnitName; + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java index 39a5e0215..85a415717 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java @@ -10,7 +10,10 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessErrorConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -29,31 +32,23 @@ public ExpUnitContextService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; } - /** - * Retrieves reference Observation Units based on a set of reference Observation Unit IDs and a Program. - * Constructs DeltaBreed observation unit source for external references and sets up pending Observation Units. - * - * @param referenceOUIds A set of reference Observation Unit IDs to retrieve - * @param program The Program associated with the Observation Units - * @return A Map containing pending Observation Units by their ID - * @throws ApiException if an error occurs during the process - */ + public Map> fetchReferenceObservationUnits( - Set referenceOUIds, - Program program + ImportContext importContext, + ExpUnitContext expUnitContext ) throws ApiException { Map> pendingUnitById = new HashMap<>(); try { // Retrieve reference Observation Units based on IDs List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( - new ArrayList(referenceOUIds), - program + new ArrayList(expUnitContext.getReferenceOUIds()), + importContext.getProgram() ); // Construct the DeltaBreed observation unit source for external references String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - if (referenceObsUnits.size() == referenceOUIds.size()) { + if (referenceObsUnits.size() == expUnitContext.getReferenceOUIds().size()) { for (BrAPIObservationUnit unit : referenceObsUnits) {// Iterate through reference Observation Units // Get external reference for the Observation Unit @@ -76,7 +71,7 @@ public Map> fetchReferenceObse ); } } else {// Handle case of missing Observation Units in data store - List missingIds = new ArrayList<>(referenceOUIds); + List missingIds = new ArrayList<>(expUnitContext.getReferenceOUIds()); Set fetchedIds = referenceObsUnits.stream() .filter(unit ->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).isPresent()) .map(unit->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).get().getReferenceId()) @@ -93,4 +88,23 @@ public Map> fetchReferenceObse throw new ApiException(e); } } + + private Map> mapPendingObservationUnitByName( + ExpUnitContext expUnitContext, + ImportContext importContext + ) { + Map> pendingUnitByName = new HashMap<>(); + for (Map.Entry> entry : expUnitContext.getPendingObsUnitByOUId().entrySet()) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + entry.getValue().getBrAPIObject().getStudyName(), + importContext.getProgram().getKey() + ); + String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( + entry.getValue().getBrAPIObject().getObservationUnitName(), + importContext.getProgram().getKey() + ); + pendingUnitByName.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName), entry.getValue()); + } + return pendingUnitByName; + } } From 6be417ab321082fe852e848d37636e4d482fbd7e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 2 May 2024 16:57:58 -0400 Subject: [PATCH 013/203] add common methods --- .../model/ExpUnitMiddlewareContext.java | 2 ++ .../experiment/model/ProcessedData.java | 4 ++++ .../experiment/services/StudyService.java | 19 ++++++++++++++++++- .../experiment/services/TrialService.java | 19 ++++++++++--------- 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java index 6a8f9183e..740707142 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java @@ -2,6 +2,7 @@ import lombok.*; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; @Getter @Setter @@ -9,4 +10,5 @@ public class ExpUnitMiddlewareContext { private ImportContext importContext; private ExpUnitContext expUnitContext; + private PendingData pendingData; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java index 4c8fe464d..551af741b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java @@ -1,6 +1,9 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.model; import lombok.*; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; + +import java.util.Map; @Getter @Setter @@ -8,4 +11,5 @@ @ToString @NoArgsConstructor public class ProcessedData { + Map statistics; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java index daca5630f..e3effb700 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java @@ -106,7 +106,7 @@ private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { * @throws ApiException if there is an issue fetching the studies * @throws IllegalStateException if any requested study database IDs are not found in the result set */ - private List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { + public List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); if (studies.size() != studyDbIds.size()) { List missingIds = new ArrayList<>(studyDbIds); @@ -117,4 +117,21 @@ private List fetchStudiesByDbId(Set studyDbIds, Program prog return studies; } + // TODO: used by both workflows + private void initializeStudiesForExistingObservationUnits( + Program program, + Map> studyByName + ) throws Exception { + Set studyDbIds = observationUnitByNameNoScope.values() + .stream() + .map(pio -> pio.getBrAPIObject() + .getStudyDbId()) + .collect(Collectors.toSet()); + + List studies = fetchStudiesByDbId(studyDbIds, program); + for (BrAPIStudy study : studies) { + processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); + } + } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java index 76e365cca..d4dbc3c71 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java @@ -16,6 +16,8 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -51,14 +53,13 @@ public TrialService(BrAPITrialDAO brAPITrialDAO, * @param trialByName A map containing trials by name. (will be modified in place) * */ - public void initializeTrialsForExistingObservationUnits(Program program, - Map> observationUnitByNameNoScope, - Map> trialByName) { - if(observationUnitByNameNoScope.size() > 0) { + public void initializeTrialsForExistingObservationUnits(ImportContext importContext, + PendingData pendingData) { + if(pendingData.getObservationUnitByNameNoScope().size() > 0) { Set trialDbIds = new HashSet<>(); Set studyDbIds = new HashSet<>(); - observationUnitByNameNoScope.values() + pendingData.getObservationUnitByNameNoScope().values() .forEach(pio -> { BrAPIObservationUnit existingOu = pio.getBrAPIObject(); if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { @@ -75,7 +76,7 @@ public void initializeTrialsForExistingObservationUnits(Program program, //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId if(!studyDbIds.isEmpty()) { try { - trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, program)); + trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, importContext.getProgram())); } catch (ApiException e) { log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -83,14 +84,14 @@ public void initializeTrialsForExistingObservationUnits(Program program, } try { - List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); + List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, importContext.getProgram()); if (trials.size() != trialDbIds.size()) { List missingIds = new ArrayList<>(trialDbIds); missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); } - trials.forEach(trial -> processAndCacheTrial(trial, program, trialByName)); + trials.forEach(trial -> processAndCacheTrial(trial, importContext.getProgram(), pendingData.getTrialByNameNoScope())); } catch (ApiException e) { log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -109,7 +110,7 @@ public void initializeTrialsForExistingObservationUnits(Program program, */ private Set fetchTrialDbidsForStudies(Set studyDbIds, Program program) throws ApiException { Set trialDbIds = new HashSet<>(); - List studies = fetchStudiesByDbId(studyDbIds, program); + List studies = studyService.fetchStudiesByDbId(studyDbIds, program); studies.forEach(study -> { if (StringUtils.isBlank(study.getTrialDbId())) { throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); From 82d02e61b16eb28535c94922d8d7e8d960e4559e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 3 May 2024 09:40:08 -0400 Subject: [PATCH 014/203] add service methods for getting existing brapi data --- .../experiment/services/DatasetService.java | 98 ++++++++++++++++ .../experiment/services/GermplasmService.java | 92 +++++++++++++++ .../experiment/services/LocationService.java | 110 ++++++++++++++++++ .../experiment/services/StudyService.java | 18 +++ .../experiment/services/TrialService.java | 25 ++++ 5 files changed, 343 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/GermplasmService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java new file mode 100644 index 000000000..2b5a75aad --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java @@ -0,0 +1,98 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import io.micronaut.http.server.exceptions.InternalServerException; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.BrAPIListTypes; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import java.util.*; + +public class DatasetService { + // TODO: used by expunit worflow + public Map> initializeObsVarDatasetForExistingObservationUnits( + Map> trialByName, + Program program) { + Map> obsVarDatasetByName = new HashMap<>(); + + if (trialByName.size() > 0 && + trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String datasetId = trialByName.values().iterator().next().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + + try { + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(datasetId)); + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("existing dataset summary not returned from brapi server"); + } + BrAPIListDetails dataSetDetails = brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult(); + processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + return obsVarDatasetByName; + } + + // TODO: used by create workflow + public Map> initializeObsVarDatasetByName(Program program, List experimentImportRows) { + Map> obsVarDatasetByName = new HashMap<>(); + + Optional> trialPIO = getTrialPIO(experimentImportRows); + + if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String datasetId = trialPIO.get().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + try { + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(datasetId)); + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("existing dataset summary not returned from brapi server"); + } + BrAPIListDetails dataSetDetails = brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult(); + processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + return obsVarDatasetByName; + } + + // TODO: used by expunit workflow + public Map> mapPendingObsDatasetByOUId( + String unitId, + Map> trialByOUId, + Map> obsVarDatasetByName, + Map> obsVarDatasetByOUId) { + if (!trialByOUId.isEmpty() && !obsVarDatasetByName.isEmpty() && + trialByOUId.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + obsVarDatasetByOUId.put(unitId, obsVarDatasetByName.values().iterator().next()); + } + + return obsVarDatasetByOUId; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/GermplasmService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/GermplasmService.java new file mode 100644 index 000000000..c15e71fd9 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/GermplasmService.java @@ -0,0 +1,92 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import io.micronaut.http.server.exceptions.InternalServerException; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import java.util.*; +import java.util.stream.Collectors; + +public class GermplasmService { + // TODO: used by expunit workflow + public Map> initializeGermplasmByGIDForExistingObservationUnits( + Map> unitByName, + Program program) { + Map> existingGermplasmByGID = new HashMap<>(); + + List existingGermplasms = new ArrayList<>(); + if(unitByName.size() > 0) { + Set germplasmDbIds = unitByName.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + try { + existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + existingGermplasms.forEach(existingGermplasm -> { + BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); + }); + return existingGermplasmByGID; + } + + // TODO: used by create worflow + public Map> initializeExistingGermplasmByGID(Program program, List experimentImportRows) { + Map> existingGermplasmByGID = new HashMap<>(); + + List existingGermplasms = new ArrayList<>(); + if(observationUnitByNameNoScope.size() > 0) { + Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + try { + existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueGermplasmGIDs = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getGid) + .distinct() + .collect(Collectors.toList()); + + try { + existingGermplasms.addAll(this.getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + existingGermplasms.forEach(existingGermplasm -> { + BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); + }); + return existingGermplasmByGID; + } + + // TODO: used by expunit workflow + public Map> mapGermplasmByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> germplasmByName, + Map> germplasmByOUId) { + String gid = unit.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.GID).getAsString(); + germplasmByOUId.put(unitId, germplasmByName.get(gid)); + + return germplasmByOUId; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java new file mode 100644 index 000000000..174150992 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java @@ -0,0 +1,110 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +@Singleton +@Slf4j +public class LocationService { + + // used by expunit workflow + public Map> initializeLocationByName( + Program program, + Map> studyByName) { + Map> locationByName = new HashMap<>(); + + List existingLocations = new ArrayList<>(); + if(studyByName.size() > 0) { + Set locationDbIds = studyByName.values() + .stream() + .map(study -> study.getBrAPIObject() + .getLocationDbId()) + .collect(Collectors.toSet()); + try { + existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + existingLocations.forEach(existingLocation -> locationByName.put( + existingLocation.getName(), + new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()) + ) + ); + return locationByName; + } + + // TODO: used by create workflow + public Map> initializeUniqueLocationNames(Program program, List experimentImportRows) { + Map> locationByName = new HashMap<>(); + + List existingLocations = new ArrayList<>(); + if(studyByNameNoScope.size() > 0) { + Set locationDbIds = studyByNameNoScope.values() + .stream() + .map(study -> study.getBrAPIObject() + .getLocationDbId()) + .collect(Collectors.toSet()); + try { + existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueLocationNames = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getEnvLocation) + .distinct() + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + try { + existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); + return locationByName; + } + + // TODO: used by expunit workflow + public Map> mapPendingLocationByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> studyByOUId, + Map> locationByName, + Map> locationByOUId + ) { + if (unit.getLocationName() != null) { + locationByOUId.put(unitId, locationByName.get(unit.getLocationName())); + } else if (studyByOUId.get(unitId) != null && studyByOUId.get(unitId).getBrAPIObject().getLocationName() != null) { + locationByOUId.put( + unitId, + locationByName.get(studyByOUId.get(unitId).getBrAPIObject().getLocationName()) + ); + } else { + throw new IllegalStateException("Observation unit missing location: " + unitId); + } + + return locationByOUId; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java index e3effb700..b69e805c8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java @@ -9,6 +9,7 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPISeason; import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -134,4 +135,21 @@ private void initializeStudiesForExistingObservationUnits( } } + // TODO: used by expunit workflow + public Map> mapPendingStudyByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> studyByName, + Map> studyByOUId, + Program program + ) { + if (unit.getStudyName() != null) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); + studyByOUId.put(unitId, studyByName.get(studyName)); + } else { + throw new IllegalStateException("Observation unit missing study name: " + unitId); + } + + return studyByOUId; + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java index d4dbc3c71..28f536081 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java @@ -180,4 +180,29 @@ private Map> initializeTrialByNameNoScop return trialByName; } + // TODO: used by expunit workflow + public Map> mapPendingTrialByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> trialByName, + Map> studyByName, + Map> trialByOUId, + Program program + ) { + String trialName; + if (unit.getTrialName() != null) { + trialName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getTrialName(), program.getKey()); + } else if (unit.getStudyName() != null) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); + trialName = Utilities.removeProgramKeyAndUnknownAdditionalData( + studyByName.get(studyName).getBrAPIObject().getTrialName(), + program.getKey() + ); + } else { + throw new IllegalStateException("Observation unit missing trial name and study name: " + unitId); + } + trialByOUId.put(unitId, trialByName.get(trialName)); + + return trialByOUId; + } } From 1ca976d9abec83e50d42c9a1f4e8e47d8d98229b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 3 May 2024 09:49:39 -0400 Subject: [PATCH 015/203] add overloaded trial methods for fetching trial pio --- .../experiment/services/TrialService.java | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java index 28f536081..6c07df8f7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java @@ -16,14 +16,19 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; +import java.math.BigInteger; import java.util.*; +import java.util.function.Supplier; import java.util.stream.Collectors; @Singleton @@ -205,4 +210,52 @@ public Map> mapPendingTrialByOUId( return trialByOUId; } + + // TODO: overloaded method used by expunit workflow + public PendingImportObject fetchOrCreateTrialPIO( + ImportContext importContext, + ExpUnitContext expUnitContext + ) throws UnprocessableEntityException { + PendingImportObject trialPio; + + + + trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); + + + return trialPio; + } + + // TODO: overloaded method used by create workflow + private PendingImportObject fetchOrCreateTrialPIO( + ImportContext importContext + ) throws UnprocessableEntityException { + PendingImportObject trialPio; + + + if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { + PendingImportObject envPio; + trialPio = trialByNameNoScope.get(importRow.getExpTitle()); + envPio = studyByNameNoScope.get(importRow.getEnv()); + + // creating new units for existing experiments and environments is not possible + if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && + (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ + throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); + } + } else if (!trialByNameNoScope.isEmpty()) { + throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); + } else { + UUID id = UUID.randomUUID(); + String expSeqValue = null; + if (commit) { + expSeqValue = expNextVal.get().toString(); + } + BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); + trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); + trialByNameNoScope.put(importRow.getExpTitle(), trialPio); + } + + return trialPio; + } } From b4906a54fbaaa790dfcbad3b1b768c9afaad5b6f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 3 May 2024 10:08:10 -0400 Subject: [PATCH 016/203] add pending data context to signature --- .../experiment/ExperimentUtilities.java | 16 ++++++++++++++++ .../experiment/services/TrialService.java | 6 ++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 92d333ce9..1c55f7545 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -2,8 +2,10 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; public class ExperimentUtilities { @@ -25,4 +27,18 @@ public static String createObservationUnitKey(String studyName, String obsUnitNa return studyName + obsUnitName; } + /** + * Returns the single value from the given map, throwing an UnprocessableEntityException if the map does not contain exactly one entry. + * + * @param map The map from which to retrieve the single value. + * @param message The error message to include in the UnprocessableEntityException if the map does not contain exactly one entry. + * @return The single value from the map. + * @throws UnprocessableEntityException if the map does not contain exactly one entry. + */ + public V getSingleEntryValue(Map map, String message) throws UnprocessableEntityException { + if (map.size() != 1) { + throw new UnprocessableEntityException(message); + } + return map.values().iterator().next(); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java index 6c07df8f7..83b36f9d7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java @@ -214,6 +214,7 @@ public Map> mapPendingTrialByOUId( // TODO: overloaded method used by expunit workflow public PendingImportObject fetchOrCreateTrialPIO( ImportContext importContext, + PendingData pendingData, ExpUnitContext expUnitContext ) throws UnprocessableEntityException { PendingImportObject trialPio; @@ -227,8 +228,9 @@ public PendingImportObject fetchOrCreateTrialPIO( } // TODO: overloaded method used by create workflow - private PendingImportObject fetchOrCreateTrialPIO( - ImportContext importContext + public PendingImportObject fetchOrCreateTrialPIO( + ImportContext importContext, + PendingData pendingData ) throws UnprocessableEntityException { PendingImportObject trialPio; From 8f15d08c985a650b0c09ef045ed76ce5e115b6b8 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 3 May 2024 11:04:37 -0400 Subject: [PATCH 017/203] add ValidateService --- .../experiment/ExperimentUtilities.java | 18 +++ .../experiment/services/DatasetService.java | 90 +++++++++++++ .../experiment/services/LocationService.java | 26 ++++ .../services/ObservationUnitService.java | 127 ++++++++++++++++++ .../experiment/services/StudyService.java | 102 ++++++++++++++ .../experiment/services/ValidateService.java | 112 +++++++++++++++ 6 files changed, 475 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 1c55f7545..25b87a0e8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -1,7 +1,11 @@ package org.breedinginsight.brapps.importer.services.processors.experiment; +import com.google.gson.JsonObject; +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import java.util.List; @@ -41,4 +45,18 @@ public V getSingleEntryValue(Map map, String message) throws Unproc } return map.values().iterator().next(); } + + /* + * this will add the given year to the additionalInfo field of the BrAPIStudy (if it does not already exist) + * */ + public void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study, String year) { + JsonObject additionalInfo = study.getAdditionalInfo(); + if (additionalInfo==null){ + additionalInfo = new JsonObject(); + study.setAdditionalInfo(additionalInfo); + } + if( additionalInfo.get(BrAPIAdditionalInfoFields.ENV_YEAR)==null) { + additionalInfo.addProperty(BrAPIAdditionalInfoFields.ENV_YEAR, year); + } + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java index 2b5a75aad..db5fcdedb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java @@ -8,9 +8,15 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; import java.util.*; @@ -95,4 +101,88 @@ public Map> mapPendingObsDatasetBy return obsVarDatasetByOUId; } + + // TODO: used by both workflows + public void addObsVarsToDatasetDetails(PendingImportObject pio, List referencedTraits, Program program) { + BrAPIListDetails details = pio.getBrAPIObject(); + referencedTraits.forEach(trait -> { + String id = Utilities.appendProgramKey(trait.getObservationVariableName(), program.getKey()); + + // TODO - Don't append the key if connected to a brapi service operating with legacy data(no appended program key) + + if (!details.getData().contains(id) && ImportObjectState.EXISTING != pio.getState()) { + details.getData().add(id); + } + if (!details.getData().contains(id) && ImportObjectState.EXISTING == pio.getState()) { + details.getData().add(id); + pio.setState(ImportObjectState.MUTATED); + } + }); + } + + // TODO: used by expunit workflow + public void fetchOrCreateDatasetPIO(ImportContext importContext, + PendingData pendingData, + ExpUnitContext expUnitContext, + List referencedTraits) throws UnprocessableEntityException { + PendingImportObject pio; + PendingImportObject trialPIO = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); + String name = String.format("Observation Dataset [%s-%s]", + program.getKey(), + trialPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString()); + if (obsVarDatasetByName.containsKey(name)) { + pio = obsVarDatasetByName.get(name); + } else { + UUID id = UUID.randomUUID(); + BrAPIListDetails newDataset = importRow.constructDatasetDetails( + name, + id, + BRAPI_REFERENCE_SOURCE, + program, + trialPIO.getId().toString()); + pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); + trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); + if (ImportObjectState.EXISTING == trialPIO.getState()) { + trialPIO.setState(ImportObjectState.MUTATED); + } + obsVarDatasetByName.put(name, pio); + } + addObsVarsToDatasetDetails(pio, referencedTraits, program); + } + + // TODO: used by create workflow + public void fetchOrCreateDatasetPIO(ImportContext importContext, + PendingData pendingData, + List referencedTraits) throws UnprocessableEntityException { + PendingImportObject pio; + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); + + String name = String.format("Observation Dataset [%s-%s]", + program.getKey(), + trialPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString()); + if (obsVarDatasetByName.containsKey(name)) { + pio = obsVarDatasetByName.get(name); + } else { + UUID id = UUID.randomUUID(); + BrAPIListDetails newDataset = importRow.constructDatasetDetails( + name, + id, + BRAPI_REFERENCE_SOURCE, + program, + trialPIO.getId().toString()); + pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); + trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); + if (ImportObjectState.EXISTING == trialPIO.getState()) { + trialPIO.setState(ImportObjectState.MUTATED); + } + obsVarDatasetByName.put(name, pio); + } + addObsVarsToDatasetDetails(pio, referencedTraits, program); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java index 174150992..977306c7f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java @@ -9,6 +9,8 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.utilities.Utilities; @@ -107,4 +109,28 @@ public Map> mapPendingLocationByOUI return locationByOUId; } + + // TODO: used by expunit workflow + private void fetchOrCreateLocationPIO(ImportContext importContext, ExpUnitContext expUnitContext) { + PendingImportObject pio; + String envLocationName = pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName(); + if (!locationByName.containsKey((importRow.getEnvLocation()))) { + ProgramLocation newLocation = new ProgramLocation(); + newLocation.setName(envLocationName); + pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); + this.locationByName.put(envLocationName, pio); + } + } + + // TODO: used by create workflow + private void fetchOrCreateLocationPIO(ImportContext importContext) { + PendingImportObject pio; + String envLocationName = importRow.getEnvLocation(); + if (!locationByName.containsKey((importRow.getEnvLocation()))) { + ProgramLocation newLocation = new ProgramLocation(); + newLocation.setName(envLocationName); + pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); + this.locationByName.put(envLocationName, pio); + } + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java new file mode 100644 index 000000000..1e2c121cd --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java @@ -0,0 +1,127 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; + +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +public class ObservationUnitService { + // TODO: used by expUnit workflow + public PendingImportObject fetchOrCreateObsUnitPIO(ImportContext importContext, + PendingData pendingData, + ExpUnitContext expUnitContext, + String envSeqValue) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { + PendingImportObject pio; + String key = createObservationUnitKey(importRow); + if (hasAllReferenceUnitIds) { + pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); + } else if (observationUnitByNameNoScope.containsKey(key)) { + pio = observationUnitByNameNoScope.get(key); + } else { + String germplasmName = ""; + if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { + germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) + .getBrAPIObject() + .getGermplasmName(); + } + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; + UUID trialID = trialPIO.getId(); + UUID datasetId = null; + if (commit) { + datasetId = UUID.fromString(trialPIO.getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + } + PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); + + // check for existing units if this is an existing study + if (studyPIO.getBrAPIObject().getStudyDbId() != null) { + List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); + List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); + if (matchingOU.isEmpty()) { + throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); + } else { + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); + } + } else { + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); + } + this.observationUnitByNameNoScope.put(key, pio); + } + return pio; + } + + // TODO: used by create workflow + public PendingImportObject fetchOrCreateObsUnitPIO(ImportContext importContext, + PendingData pendingData, + String envSeqValue) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { + PendingImportObject pio; + String key = createObservationUnitKey(importRow); + if (hasAllReferenceUnitIds) { + pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); + } else if (observationUnitByNameNoScope.containsKey(key)) { + pio = observationUnitByNameNoScope.get(key); + } else { + String germplasmName = ""; + if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { + germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) + .getBrAPIObject() + .getGermplasmName(); + } + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; + UUID trialID = trialPIO.getId(); + UUID datasetId = null; + if (commit) { + datasetId = UUID.fromString(trialPIO.getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + } + PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); + + // check for existing units if this is an existing study + if (studyPIO.getBrAPIObject().getStudyDbId() != null) { + List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); + List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); + if (matchingOU.isEmpty()) { + throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); + } else { + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); + } + } else { + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); + } + this.observationUnitByNameNoScope.put(key, pio); + } + return pio; + } + + // TODO: used by both workflows + public String createObservationUnitKey(ExperimentObservation importRow) { + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); + } + + // TODO: used by both workflows + public String createObservationUnitKey(String studyName, String obsUnitName) { + return studyName + obsUnitName; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java index b69e805c8..38ae4d9f9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java @@ -9,19 +9,26 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPISeason; import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; +import java.math.BigInteger; import java.util.*; +import java.util.function.Supplier; import java.util.stream.Collectors; @Singleton @@ -152,4 +159,99 @@ public Map> mapPendingStudyByOUId( return studyByOUId; } + + // TODO: used by expunit workflow + private PendingImportObject fetchOrCreateStudyPIO( + ImportContext importContext, + ExpUnitContext expUnitContext, + String expSequenceValue, + Supplier envNextVal + ) throws UnprocessableEntityException { + PendingImportObject pio; + if (hasAllReferenceUnitIds) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), + program.getKey() + ); + pio = studyByNameNoScope.get(studyName); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { + pio = studyByNameNoScope.get(importRow.getEnv()); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else { + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + UUID trialID = trialPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); + newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies + + // It is assumed that the study has only one season, And that the Years and not + // the dbId's are stored in getSeason() list. + String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season + if (commit) { + if(StringUtils.isNotBlank(year)) { + String seasonID = this.yearToSeasonDbId(year, program.getId()); + newStudy.setSeasons(Collections.singletonList(seasonID)); + } + } else { + addYearToStudyAdditionalInfo(program, newStudy, year); + } + + pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); + this.studyByNameNoScope.put(importRow.getEnv(), pio); + } + return pio; + } + + // TODO: used by create workflow + private PendingImportObject fetchOrCreateStudyPIO( + ImportContext importContext, + String expSequenceValue, + Supplier envNextVal + ) throws UnprocessableEntityException { + PendingImportObject pio; + if (hasAllReferenceUnitIds) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), + program.getKey() + ); + pio = studyByNameNoScope.get(studyName); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { + pio = studyByNameNoScope.get(importRow.getEnv()); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else { + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + UUID trialID = trialPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); + newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies + + // It is assumed that the study has only one season, And that the Years and not + // the dbId's are stored in getSeason() list. + String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season + if (commit) { + if(StringUtils.isNotBlank(year)) { + String seasonID = this.yearToSeasonDbId(year, program.getId()); + newStudy.setSeasons(Collections.singletonList(seasonID)); + } + } else { + addYearToStudyAdditionalInfo(program, newStudy, year); + } + + pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); + this.studyByNameNoScope.put(importRow.getEnv(), pio); + } + return pio; + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java new file mode 100644 index 000000000..a84b84e30 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java @@ -0,0 +1,112 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import tech.tablesaw.columns.Column; + +import java.util.List; +import java.util.Map; + +public class ValidateService { + // TODO: used by expUnit workflow + public void prepareDataForValidation(ImportContext importContext, + ExpUnitContext expUnitContext, + List> phenotypeCols) { + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); + List> observations = mappedImportRow.getObservations(); + String observationHash; + if (hasAllReferenceUnitIds) { + String refOUId = importRow.getObsUnitID(); + mappedImportRow.setTrial(pendingTrialByOUId.get(refOUId)); + mappedImportRow.setLocation(pendingLocationByOUId.get(refOUId)); + mappedImportRow.setStudy(pendingStudyByOUId.get(refOUId)); + mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(refOUId)); + mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(refOUId)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + observationHash = getObservationHash( + pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + + pendingObsUnitByOUId.get(refOUId).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(column), + pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + ); + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(observationHash)); + } + + } else { + mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); + mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); + mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); + mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); + mappedImportRow.setGermplasm(getGidPIO(importRow)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); + } + } + + mappedBrAPIImport.put(rowNum, mappedImportRow); + } + } + + // TODO: used by create workflow + public void prepareDataForValidation(ImportContext importContext, + List> phenotypeCols) { + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); + List> observations = mappedImportRow.getObservations(); + String observationHash; + if (hasAllReferenceUnitIds) { + String refOUId = importRow.getObsUnitID(); + mappedImportRow.setTrial(pendingTrialByOUId.get(refOUId)); + mappedImportRow.setLocation(pendingLocationByOUId.get(refOUId)); + mappedImportRow.setStudy(pendingStudyByOUId.get(refOUId)); + mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(refOUId)); + mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(refOUId)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + observationHash = getObservationHash( + pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + + pendingObsUnitByOUId.get(refOUId).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(column), + pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + ); + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(observationHash)); + } + + } else { + mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); + mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); + mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); + mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); + mappedImportRow.setGermplasm(getGidPIO(importRow)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); + } + } + + mappedBrAPIImport.put(rowNum, mappedImportRow); + } + } +} From 63091c3ccff9c83972f903e68fb418b1d5f7aa64 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 3 May 2024 12:48:22 -0400 Subject: [PATCH 018/203] add ValidateService#validateFields --- .../experiment/create/model/PendingData.java | 2 + .../experiment/services/ValidateService.java | 66 +++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java index c3f769a71..340a39fba 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -6,6 +6,7 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.model.ProgramLocation; @@ -25,4 +26,5 @@ public class PendingData { private Map> locationByName; private Map> obsVarDatasetByName; private Map> existingGermplasmByGID; + private ValidationErrors validationErrors; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java index a84b84e30..8004a5548 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java @@ -1,16 +1,25 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.services; +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.apache.commons.lang3.StringUtils; import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.model.User; import tech.tablesaw.columns.Column; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; public class ValidateService { // TODO: used by expUnit workflow @@ -109,4 +118,61 @@ public void prepareDataForValidation(ImportContext importContext, mappedBrAPIImport.put(rowNum, mappedImportRow); } } + + // TODO: used by expUnit workflow + public void validateFields(ImportContext importContext, + PendingData pendingData, + ExpUnitContext expUnitContext, + List referencedTraits, Program program, + List> phenotypeCols) { + //fetching any existing observations for any OUs in the import + CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); + for ( Trait trait: referencedTraits) { + colVarMap.put(trait.getObservationVariableName(),trait); + } + Set uniqueStudyAndObsUnit = new HashSet<>(); + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); + if (hasAllReferenceUnitIds) { + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); + } else { + if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. + validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); + } + validateTestOrCheck(importRow, validationErrors, rowNum); + validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); + validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); + } + } + } + + // TODO: used by create workflow + public void validateFields(ImportContext importContext, + PendingData pendingData, + List referencedTraits, Program program, + List> phenotypeCols) { + //fetching any existing observations for any OUs in the import + CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); + for ( Trait trait: referencedTraits) { + colVarMap.put(trait.getObservationVariableName(),trait); + } + Set uniqueStudyAndObsUnit = new HashSet<>(); + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); + if (hasAllReferenceUnitIds) { + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); + } else { + if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. + validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); + } + validateTestOrCheck(importRow, validationErrors, rowNum); + validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); + validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); + } + } + } } From 8a89c533fdbba964694cdae4b7120d6eefc56ee1 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 3 May 2024 15:36:39 -0400 Subject: [PATCH 019/203] add observation and statistics service --- .../services/ObservationService.java | 143 ++++++++++++++++++ .../services/ObservationUnitService.java | 19 +++ .../services/StatisticsService.java | 92 +++++++++++ .../experiment/services/ValidateService.java | 137 +++++++++++++++++ 4 files changed, 391 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StatisticsService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java new file mode 100644 index 000000000..287e4e595 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java @@ -0,0 +1,143 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.apache.commons.lang3.StringUtils; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Trait; +import org.breedinginsight.model.User; +import tech.tablesaw.columns.Column; + +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; + +public class ObservationService { + // TODO: used with expUnit workflow + public void validateObservations(PendingData pendingData, + int rowNum, + ImportContext importContext, + ExpUnitContext expUnitContext, + List> phenotypeCols, + CaseInsensitiveMap colVarMap) { + for (Column phenoCol : phenotypeCols) { + String importHash; + String importObsValue = phenoCol.getString(rowNum); + + importHash = getImportObservationHash( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(phenoCol), + pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() + ); + + validateObservation(importHash); + } + } + + // TODO: used with create workflow + public void validateObservations(PendingData pendingData, + int rowNum, + ImportContext importContext, + List> phenotypeCols, + CaseInsensitiveMap colVarMap) { + + + for (Column phenoCol : phenotypeCols) { + String importHash; + String importObsValue = phenoCol.getString(rowNum); + + importHash = getImportObservationHash(importRow, phenoCol.name()); + + validateObservation(importHash); + } + + } + + // TODO: used by both workflows + private void validateObservation(String importHash) { + + + String importObsValue = phenoCol.getString(rowNum); + + + // error if import observation data already exists and user has not selected to overwrite + if (commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && + this.existingObsByObsHash.containsKey(importHash) && + StringUtils.isNotBlank(phenoCol.getString(rowNum)) && + !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { + addRowError( + phenoCol.name(), + String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), + validationErrors, rowNum + ); + + // preview case where observation has already been committed and the import row ObsVar data differs from what + // had been saved prior to import + } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { + + // add a change log entry when updating the value of an observation + if (commit) { + BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; + String prior = ""; + if (isValueMatched(importHash, importObsValue)) { + prior.concat(existingObsByObsHash.get(importHash).getValue()); + } + if (timeStampColByPheno.containsKey(phenoCol.name()) && isTimestampMatched(importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { + prior = prior.isEmpty() ? prior : prior.concat(" "); + prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); + } + ChangeLogEntry change = new ChangeLogEntry(prior, + reason, + user.getId(), + timestamp + ); + + // create the changelog field in additional info if it does not already exist + if (pendingObservation.getAdditionalInfo().isJsonNull()) { + pendingObservation.setAdditionalInfo(new JsonObject()); + pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + + if (pendingObservation.getAdditionalInfo() != null && !pendingObservation.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { + pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + + // add a new entry to the changelog + pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); + } + + // preview case where observation has already been committed and import ObsVar data is the + // same as has been committed prior to import + } else if (isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { + BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); + existingObs.setObservationVariableName(phenoCol.name()); + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + observationByHash.get(importHash).setBrAPIObject(existingObs); + + // preview case where observation has already been committed and import ObsVar data is empty prior to import + } else if (!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + } else { + validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); + + //Timestamp validation + if (timeStampColByPheno.containsKey(phenoCol.name())) { + Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); + validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); + } + } + + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java index 1e2c121cd..b0719223b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java @@ -1,9 +1,11 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.services; +import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -16,6 +18,7 @@ import org.breedinginsight.utilities.Utilities; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -124,4 +127,20 @@ public String createObservationUnitKey(ExperimentObservation importRow) { public String createObservationUnitKey(String studyName, String obsUnitName) { return studyName + obsUnitName; } + + // TODO: used by create workflow + public void validateObservationUnits(ValidationErrors validationErrors, + Set uniqueStudyAndObsUnit, + int rowNum, + ExperimentObservation importRow) { + validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + + String key = createObservationUnitKey(importRow); + PendingImportObject ouPIO = observationUnitByNameNoScope.get(key); + if(ouPIO.getState() == ImportObjectState.NEW && StringUtils.isNotBlank(importRow.getObsUnitID())) { + addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "Could not find observation unit by ObsUnitDBID", validationErrors, rowNum); + } + + validateGeoCoordinates(validationErrors, rowNum, importRow); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StatisticsService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StatisticsService.java new file mode 100644 index 000000000..26e6d7f17 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StatisticsService.java @@ -0,0 +1,92 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; +import org.breedinginsight.services.exceptions.ValidatorException; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +public class StatisticsService { + // TODO: used by both workflows + public Map generateStatisticsMap(List importRows) { + // Data for stats. + HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names + HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's + HashSet gidCounter = new HashSet<>(); // set of unique GID's + + for (BrAPIImport row : importRows) { + ExperimentObservation importRow = (ExperimentObservation) row; + // Collect date for stats. + addIfNotNull(environmentNameCounter, importRow.getEnv()); + addIfNotNull(obsUnitsIDCounter, createObservationUnitKey(importRow)); + addIfNotNull(gidCounter, importRow.getGid()); + } + + int numNewObservations = Math.toIntExact( + observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + int numExistingObservations = Math.toIntExact( + this.observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.EXISTING && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + int numMutatedObservations = Math.toIntExact( + this.observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() + .newObjectCount(environmentNameCounter.size()) + .build(); + ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() + .newObjectCount(obsUnitsIDCounter.size()) + .build(); + ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() + .newObjectCount(gidCounter.size()) + .build(); + ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder() + .newObjectCount(numNewObservations) + .build(); + ImportPreviewStatistics existingObservationStats = ImportPreviewStatistics.builder() + .newObjectCount(numExistingObservations) + .build(); + ImportPreviewStatistics mutatedObservationStats = ImportPreviewStatistics.builder() + .newObjectCount(numMutatedObservations) + .build(); + + return Map.of( + "Environments", environmentStats, + "Observation_Units", obdUnitStats, + "GIDs", gidStats, + "Observations", observationStats, + "Existing_Observations", existingObservationStats, + "Mutated_Observations", mutatedObservationStats + ); + } + + // TODO: used by both workflows + public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { + // TODO + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java index 8004a5548..bb0db1662 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java @@ -7,6 +7,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; @@ -14,6 +15,7 @@ import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; import org.breedinginsight.model.User; +import org.breedinginsight.utilities.Utilities; import tech.tablesaw.columns.Column; import java.util.HashSet; @@ -175,4 +177,139 @@ public void validateFields(ImportContext importContext, } } } + + // TODO: used by create workflow + private void validateTestOrCheck(ExperimentObservation importRow, ValidationErrors validationErrors, int rowNum) { + String testOrCheck = importRow.getTestOrCheck(); + if ( ! ( testOrCheck==null || testOrCheck.isBlank() + || "C".equalsIgnoreCase(testOrCheck) || "CHECK".equalsIgnoreCase(testOrCheck) + || "T".equalsIgnoreCase(testOrCheck) || "TEST".equalsIgnoreCase(testOrCheck) ) + ){ + addRowError(ExperimentObservation.Columns.TEST_CHECK, String.format("Invalid value (%s)", testOrCheck), validationErrors, rowNum) ; + } + } + + // TODO: used by create workflow + private void validateConditionallyRequired(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, Program program, boolean commit) { + ImportObjectState expState = this.trialByNameNoScope.get(importRow.getExpTitle()) + .getState(); + ImportObjectState envState = this.studyByNameNoScope.get(importRow.getEnv()).getState(); + + String errorMessage = BLANK_FIELD_EXPERIMENT; + if (expState == ImportObjectState.EXISTING && envState == ImportObjectState.NEW) { + errorMessage = BLANK_FIELD_ENV; + } else if(expState == ImportObjectState.EXISTING && envState == ImportObjectState.EXISTING) { + errorMessage = BLANK_FIELD_OBS; + } + + if(expState == ImportObjectState.NEW || envState == ImportObjectState.NEW) { + validateRequiredCell(importRow.getGid(), ExperimentObservation.Columns.GERMPLASM_GID, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpTitle(), ExperimentObservation.Columns.EXP_TITLE,errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpUnit(), ExperimentObservation.Columns.EXP_UNIT, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpType(), ExperimentObservation.Columns.EXP_TYPE, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getEnv(), ExperimentObservation.Columns.ENV, errorMessage, validationErrors, rowNum); + if(validateRequiredCell(importRow.getEnvLocation(), ExperimentObservation.Columns.ENV_LOCATION, errorMessage, validationErrors, rowNum)) { + if(!Utilities.removeProgramKeyAndUnknownAdditionalData(this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getLocationName(), program.getKey()).equals(importRow.getEnvLocation())) { + addRowError(ExperimentObservation.Columns.ENV_LOCATION, ENV_LOCATION_MISMATCH, validationErrors, rowNum); + } + } + if(validateRequiredCell(importRow.getEnvYear(), ExperimentObservation.Columns.ENV_YEAR, errorMessage, validationErrors, rowNum)) { + String studyYear = StringUtils.defaultString( this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getSeasons().get(0) ); + String rowYear = importRow.getEnvYear(); + if(commit) { + rowYear = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); + } + if(StringUtils.isNotBlank(studyYear) && !studyYear.equals(rowYear)) { + addRowError(ExperimentObservation.Columns.ENV_YEAR, ENV_YEAR_MISMATCH, validationErrors, rowNum); + } + } + validateRequiredCell(importRow.getExpUnitId(), ExperimentObservation.Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpReplicateNo(), ExperimentObservation.Columns.REP_NUM, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpBlockNo(), ExperimentObservation.Columns.BLOCK_NUM, errorMessage, validationErrors, rowNum); + + if(StringUtils.isNotBlank(importRow.getObsUnitID())) { + addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); + } + } else { + //Check if existing environment. If so, ObsUnitId must be assigned + validateRequiredCell( + importRow.getObsUnitID(), + ExperimentObservation.Columns.OBS_UNIT_ID, + MISSING_OBS_UNIT_ID_ERROR, + validationErrors, + rowNum + ); + } + } + + // TODO: used by create workflow + public void validateGeoCoordinates(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow) { + + String lat = importRow.getLatitude(); + String lon = importRow.getLongitude(); + String elevation = importRow.getElevation(); + + // If any of Lat, Long, or Elevation are provided, Lat and Long must both be provided. + if (StringUtils.isNotBlank(lat) || StringUtils.isNotBlank(lon) || StringUtils.isNotBlank(elevation)) { + if (StringUtils.isBlank(lat)) { + addRowError(ExperimentObservation.Columns.LAT, "Latitude must be provided for complete coordinate specification", validationErrors, rowNum); + } + if (StringUtils.isBlank(lon)) { + addRowError(ExperimentObservation.Columns.LONG, "Longitude must be provided for complete coordinate specification", validationErrors, rowNum); + } + } + + // Validate coordinate values + boolean latBadValue = false; + boolean lonBadValue = false; + boolean elevationBadValue = false; + double latDouble; + double lonDouble; + double elevationDouble; + + // Only check latitude format if not blank since already had previous error + if (StringUtils.isNotBlank(lat)) { + try { + latDouble = Double.parseDouble(lat); + if (latDouble < -90 || latDouble > 90) { + latBadValue = true; + } + } catch (NumberFormatException e) { + latBadValue = true; + } + } + + // Only check longitude format if not blank since already had previous error + if (StringUtils.isNotBlank(lon)) { + try { + lonDouble = Double.parseDouble(lon); + if (lonDouble < -180 || lonDouble > 180) { + lonBadValue = true; + } + } catch (NumberFormatException e) { + lonBadValue = true; + } + } + + if (StringUtils.isNotBlank(elevation)) { + try { + elevationDouble = Double.parseDouble(elevation); + } catch (NumberFormatException e) { + elevationBadValue = true; + } + } + + if (latBadValue) { + addRowError(ExperimentObservation.Columns.LAT, "Invalid Lat value (expected range -90 to 90)", validationErrors, rowNum); + } + + if (lonBadValue) { + addRowError(ExperimentObservation.Columns.LONG, "Invalid Long value (expected range -180 to 180)", validationErrors, rowNum); + } + + if (elevationBadValue) { + addRowError(ExperimentObservation.Columns.LONG, "Invalid Elevation value (numerals expected)", validationErrors, rowNum); + } + + } } From 42b1482f9266ab220e7016f30851db1756dd1ea7 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 6 May 2024 14:15:33 -0400 Subject: [PATCH 020/203] update Middleware#link to handle nested middleware --- .../processors/experiment/middleware/Middleware.java | 10 +++++++--- .../processors/experiment/services/DatasetService.java | 2 ++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java index d2d7443b8..6d097722f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java @@ -13,9 +13,9 @@ public abstract class Middleware { public static Middleware link(Middleware first, Middleware... chain) { Middleware head = first; for (Middleware nextInChain: chain) { - nextInChain.prior = head; - head.next = nextInChain; - head = nextInChain; + nextInChain.prior = head.getLastLink(); + head.getLastLink().next = nextInChain; + head = nextInChain.getLastLink(); } return first; } @@ -49,4 +49,8 @@ protected boolean compensatePrior(T context, MiddlewareError error) { } return prior.compensate(context, error); } + + private Middleware getLastLink() { + return this.next == null ? this : this.next.getLastLink(); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java index db5fcdedb..83bf01381 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java @@ -185,4 +185,6 @@ public void fetchOrCreateDatasetPIO(ImportContext importContext, } addObsVarsToDatasetDetails(pio, referencedTraits, program); } + + } From 7ca5817dad0fc2b26dea8a0e2bd421510bb1b102 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 6 May 2024 15:41:18 -0400 Subject: [PATCH 021/203] add method to create new brapi trials for import --- .../postBrAPIData/CreateBrAPITrials.java | 4 +++ .../services/ObservationService.java | 26 +++++++++++++++++++ .../services/ObservationUnitService.java | 21 +++++++++++++++ .../experiment/services/TrialService.java | 15 +++++++++++ 4 files changed, 66 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/postBrAPIData/CreateBrAPITrials.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/postBrAPIData/CreateBrAPITrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/postBrAPIData/CreateBrAPITrials.java new file mode 100644 index 000000000..b84b0bd3e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/postBrAPIData/CreateBrAPITrials.java @@ -0,0 +1,4 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.postBrAPIData; + +public class CreateBrAPITrials { +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java index 287e4e595..d57eca250 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java @@ -10,9 +10,11 @@ import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; import org.breedinginsight.model.User; import tech.tablesaw.columns.Column; @@ -140,4 +142,28 @@ private void validateObservation(String importHash) { } } + + // TODO: used by both workflows + private void updateObservationDependencyValues(Program program) { + String programKey = program.getKey(); + + // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds + this.observationUnitByNameNoScope.values().stream() + .map(PendingImportObject::getBrAPIObject) + .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); + + // Update ObservationVariable DbIds + List traits = getTraitList(program); + CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); + for ( Trait trait: traits) { + traitMap.put(trait.getObservationVariableName(),trait); + } + for (PendingImportObject observation : this.observationByHash.values()) { + String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); + if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { + String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); + observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); + } + } + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java index b0719223b..510ef75b1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java @@ -18,6 +18,7 @@ import org.breedinginsight.utilities.Utilities; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -143,4 +144,24 @@ public void validateObservationUnits(ValidationErrors validationErrors, validateGeoCoordinates(validationErrors, rowNum, importRow); } + + // TODO: used by both workflows + private void updateObsUnitDependencyValues(String programKey) { + + // update study DbIds + this.studyByNameNoScope.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(study -> updateStudyDbId(study, programKey)); + + // update germplasm DbIds + this.existingGermplasmByGID.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(this::updateGermplasmDbId); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java index 83b36f9d7..325347d0c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java @@ -15,6 +15,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; @@ -260,4 +261,18 @@ public PendingImportObject fetchOrCreateTrialPIO( return trialPio; } + + // TODO: used by both workflows + List commitNewPendingTrialsToBrAPIStore(ImportContext context, PendingData pendingData) { + List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); + List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newTrials, program.getId(), upload)); + // set the DbId to the for each newly created trial + for (BrAPITrial createdTrial : createdTrials) { + String createdTrialName = Utilities.removeProgramKey(createdTrial.getTrialName(), program.getKey()); + this.trialByNameNoScope.get(createdTrialName) + .getBrAPIObject() + .setTrialDbId(createdTrial.getTrialDbId()); + } + return createdTrials; + } } From afc5e918942dd6c38125e7ff117cb10cb89af1cb Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 6 May 2024 15:47:44 -0400 Subject: [PATCH 022/203] add method to update pending trials --- .../experiment/services/TrialService.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java index 325347d0c..749599179 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java @@ -275,4 +275,24 @@ List commitNewPendingTrialsToBrAPIStore(ImportContext context, Pendi } return createdTrials; } + + List commitUpdatedPendingTrialsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedTrials = new ArrayList<>(); + Map mutatedTrialsById = ProcessorData + .getMutationsByObjectId(trialByNameNoScope, BrAPITrial::getTrialDbId); + for (Map.Entry entry : mutatedTrialsById.entrySet()) { + String id = entry.getKey(); + BrAPITrial trial = entry.getValue(); + try { + updatedTrials.add(brapiTrialDAO.updateBrAPITrial(id, trial, program.getId())); + } catch (ApiException e) { + log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating dataset observation variables: ", e); + throw new InternalServerException(e.getMessage(), e); + } + } + return updatedTrials; + } } From c926da4ec16c8c7e76a96be7abff9f4bdabc66a3 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 6 May 2024 16:03:16 -0400 Subject: [PATCH 023/203] add methods for creating and updating pending datasets --- .../experiment/services/DatasetService.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java index 83bf01381..372afe408 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java @@ -5,12 +5,14 @@ import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIListTypes; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; @@ -20,6 +22,7 @@ import org.breedinginsight.utilities.Utilities; import java.util.*; +import java.util.stream.Collectors; public class DatasetService { // TODO: used by expunit worflow @@ -186,5 +189,50 @@ public void fetchOrCreateDatasetPIO(ImportContext importContext, addObsVarsToDatasetDetails(pio, referencedTraits, program); } + // TODO: used by both workflows + List commitNewPendingDatasetsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List newDatasetRequests = ProcessorData.getNewObjects(obsVarDatasetByName).stream().map(details -> { + BrAPIListNewRequest request = new BrAPIListNewRequest(); + request.setListName(details.getListName()); + request.setListType(details.getListType()); + request.setExternalReferences(details.getExternalReferences()); + request.setAdditionalInfo(details.getAdditionalInfo()); + request.data(details.getData()); + return request; + }).collect(Collectors.toList()); + List createdDatasets = new ArrayList<>(brAPIListDAO.createBrAPILists(newDatasetRequests, program.getId(), upload)); + for (BrAPIListSummary summary : createdDatasets) { + obsVarDatasetByName.get(summary.getListName()).getBrAPIObject().setListDbId(summary.getListDbId()); + } + return createdDatasets; + } + // TODO: used by both workflows + List commitUpdatedPendingDatasetToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedDatasets = new ArrayList<>(); + Map datasetNewDataById = ProcessorData + .getMutationsByObjectId(obsVarDatasetByName, BrAPIListSummary::getListDbId); + for (Map.Entry entry : datasetNewDataById.entrySet()) { + String id = entry.getKey(); + BrAPIListDetails dataset = entry.getValue(); + try { + List existingObsVarIds = brAPIListDAO.getListById(id, program.getId()).getResult().getData(); + List newObsVarIds = dataset + .getData() + .stream() + .filter(obsVarId -> !existingObsVarIds.contains(obsVarId)).collect(Collectors.toList()); + List obsVarIds = new ArrayList<>(existingObsVarIds); + obsVarIds.addAll(newObsVarIds); + dataset.setData(obsVarIds); + updatedDatasets.add(brAPIListDAO.updateBrAPIList(id, dataset, program.getId())); + } catch (ApiException e) { + log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating dataset observation variables: ", e); + throw new InternalServerException(e.getMessage(), e); + } + } + return updatedDatasets; + } } From 6ae624e76255ca75569d3e5c17b738f32ab46ea9 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 6 May 2024 16:54:23 -0400 Subject: [PATCH 024/203] add method to update brapi study dependencies --- .../experiment/services/DatasetService.java | 4 +- .../experiment/services/LocationService.java | 36 ++++++++++++++++++ .../experiment/services/StudyService.java | 38 +++++++++++++++++++ .../experiment/services/TrialService.java | 4 +- 4 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java index 372afe408..2a7cd2897 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java @@ -190,7 +190,7 @@ public void fetchOrCreateDatasetPIO(ImportContext importContext, } // TODO: used by both workflows - List commitNewPendingDatasetsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + public List commitNewPendingDatasetsToBrAPIStore(ImportContext importContext, PendingData pendingData) { List newDatasetRequests = ProcessorData.getNewObjects(obsVarDatasetByName).stream().map(details -> { BrAPIListNewRequest request = new BrAPIListNewRequest(); request.setListName(details.getListName()); @@ -208,7 +208,7 @@ List commitNewPendingDatasetsToBrAPIStore(ImportContext import } // TODO: used by both workflows - List commitUpdatedPendingDatasetToBrAPIStore(ImportContext importContext, PendingData pendingData) { + public List commitUpdatedPendingDatasetsToBrAPIStore(ImportContext importContext, PendingData pendingData) { List updatedDatasets = new ArrayList<>(); Map datasetNewDataById = ProcessorData .getMutationsByObjectId(obsVarDatasetByName, BrAPIListSummary::getListDbId); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java index 977306c7f..29e2c851d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java @@ -4,12 +4,19 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.request.BrAPIListNewRequest; +import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; @@ -133,4 +140,33 @@ private void fetchOrCreateLocationPIO(ImportContext importContext) { this.locationByName.put(envLocationName, pio); } } + + // TODO: used by both workflows + public List commitNewPendingLocationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + AuthenticatedUser actingUser = new AuthenticatedUser(upload.getUpdatedByUser().getName(), new ArrayList<>(), upload.getUpdatedByUser().getId(), new ArrayList<>()); + + List newLocations = ProcessorData.getNewObjects(this.locationByName) + .stream() + .map(location -> ProgramLocationRequest.builder() + .name(location.getName()) + .build()) + .collect(Collectors.toList()); + + List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); + // set the DbId to the for each newly created location + for (ProgramLocation createdLocation : createdLocations) { + String createdLocationName = createdLocation.getName(); + this.locationByName.get(createdLocationName) + .getBrAPIObject() + .setLocationDbId(createdLocation.getLocationDbId()); + } + return createdLocations; + } + + // TODO: used by both workflows + public List commitUpdatedPendingLocationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedLocations = new ArrayList<>(); + + return updatedLocations; + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java index 38ae4d9f9..1a5c61b05 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java @@ -13,6 +13,7 @@ import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -21,6 +22,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; @@ -254,4 +256,40 @@ private PendingImportObject fetchOrCreateStudyPIO( } return pio; } + + public void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { + // update location DbIds in studies for all distinct locations + mappedBrAPIImport.values() + .stream() + .map(PendingImport::getLocation) + .forEach(this::updateStudyLocationDbId); + + // update trial DbIds in studies for all distinct trials + this.trialByNameNoScope.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(trial -> this.updateTrialDbId(trial, programKey)); + } + + private void updateStudyLocationDbId(PendingImportObject location) { + this.studyByNameNoScope.values() + .stream() + .filter(study -> location.getId().toString() + .equals(study.getBrAPIObject() + .getLocationDbId())) + .forEach(study -> study.getBrAPIObject() + .setLocationDbId(location.getBrAPIObject().getLocationDbId())); + } + + private void updateTrialDbId(BrAPITrial trial, String programKey) { + this.studyByNameNoScope.values() + .stream() + .filter(study -> study.getBrAPIObject() + .getTrialName() + .equals(Utilities.removeProgramKey(trial.getTrialName(), programKey))) + .forEach(study -> study.getBrAPIObject() + .setTrialDbId(trial.getTrialDbId())); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java index 749599179..57ec2d52b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java @@ -263,7 +263,7 @@ public PendingImportObject fetchOrCreateTrialPIO( } // TODO: used by both workflows - List commitNewPendingTrialsToBrAPIStore(ImportContext context, PendingData pendingData) { + public List commitNewPendingTrialsToBrAPIStore(ImportContext context, PendingData pendingData) { List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newTrials, program.getId(), upload)); // set the DbId to the for each newly created trial @@ -276,7 +276,7 @@ List commitNewPendingTrialsToBrAPIStore(ImportContext context, Pendi return createdTrials; } - List commitUpdatedPendingTrialsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + public List commitUpdatedPendingTrialsToBrAPIStore(ImportContext importContext, PendingData pendingData) { List updatedTrials = new ArrayList<>(); Map mutatedTrialsById = ProcessorData .getMutationsByObjectId(trialByNameNoScope, BrAPITrial::getTrialDbId); From 314d0b129d6f563b2b2bf34cbebe51f8fb7bf0a5 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 6 May 2024 17:07:01 -0400 Subject: [PATCH 025/203] Added db migration to create workflows --- .../V1.22.0__add_experiment_workflows.sql | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql diff --git a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql new file mode 100644 index 000000000..a78947c38 --- /dev/null +++ b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql @@ -0,0 +1,41 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE importer_mapping_workflow +( + like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, + mapping_id UUID NOT NULL, + name TEXT +); + +ALTER TABLE importer_mapping_workflow + ADD FOREIGN KEY (mapping_id) REFERENCES importer_mapping (id); + +DO +$$ +DECLARE + exp_mapping_id UUID; +BEGIN + exp_mapping_id := (SELECT id FROM importer_mapping WHERE name = 'ExperimentsTemplateMap'); + +INSERT INTO public.importer_mapping_workflow (mapping_id, name) +VALUES + (exp_mapping_id, 'Create new experiment'), + (exp_mapping_id, 'Append experimental dataset'), + (exp_mapping_id, 'Create new experimental environment'); +END +$$; \ No newline at end of file From 17ec46143393cd65123b721cbb3e6b2bf971cc90 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 6 May 2024 17:15:45 -0400 Subject: [PATCH 026/203] add methods for committing studies --- .../experiment/services/StudyService.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java index 1a5c61b05..6736a7c28 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java @@ -2,6 +2,7 @@ import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.server.exceptions.InternalServerException; import io.reactivex.functions.Function; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -18,8 +19,10 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; @@ -257,7 +260,7 @@ private PendingImportObject fetchOrCreateStudyPIO( return pio; } - public void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { + private void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { // update location DbIds in studies for all distinct locations mappedBrAPIImport.values() .stream() @@ -292,4 +295,20 @@ private void updateTrialDbId(BrAPITrial trial, String programKey) { .forEach(study -> study.getBrAPIObject() .setTrialDbId(trial.getTrialDbId())); } + + // TODO: used by both workflows + public List commitNewPendingStudiessToBrAPIStore(ImportContext context, PendingData pendingData) { + List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); + updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); + List createdStudies = brAPIStudyDAO.createBrAPIStudies(newStudies, program.getId(), upload); + + return createdStudies; + } + + // TODO: used by both workflows + public List commitUpdatedPendingStudiesToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedStudies = new ArrayList<>(); + + return updatedStudies; + } } From 968e9baf363301d5e0b55561e8cb694b9d783f1b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 6 May 2024 17:33:15 -0400 Subject: [PATCH 027/203] add methods for committing obs units --- .../services/ObservationUnitService.java | 80 ++++++++++++++++++- .../experiment/services/StudyService.java | 10 ++- 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java index 510ef75b1..8dbcbc2d2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java @@ -4,23 +4,23 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.UUID; +import java.util.*; import java.util.stream.Collectors; public class ObservationUnitService { @@ -164,4 +164,76 @@ private void updateObsUnitDependencyValues(String programKey) { .map(PendingImportObject::getBrAPIObject) .forEach(this::updateGermplasmDbId); } + + // TODO: used by both workflows + public List commitNewPendingObservationUnitsToBrAPIStore(ImportContext context, PendingData pendingData) { + List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); + updateObsUnitDependencyValues(program.getKey()); + List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); + + // set the DbId to the for each newly created Observation Unit + for (BrAPIObservationUnit createdObservationUnit : createdObservationUnits) { + // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope + String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getStudyName(), program.getKey()); + String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getObservationUnitName(), program.getKey()); + String createdObsUnit_key = createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); + this.observationUnitByNameNoScope.get(createdObsUnit_key) + .getBrAPIObject() + .setObservationUnitDbId(createdObservationUnit.getObservationUnitDbId()); + } + + return createdObservationUnits; + } + + // TODO: used by both workflows + public List commitUpdatedPendingObservationUnitToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedUnits = new ArrayList<>(); + + return updatedUnits; + } + + private void updateObsUnitDependencyValues(String programKey) { + + // update study DbIds + this.studyByNameNoScope.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(study -> updateStudyDbId(study, programKey)); + + // update germplasm DbIds + this.existingGermplasmByGID.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(this::updateGermplasmDbId); + } + + private void updateStudyDbId(BrAPIStudy study, String programKey) { + this.observationUnitByNameNoScope.values() + .stream() + .filter(obsUnit -> obsUnit.getBrAPIObject() + .getStudyName() + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), programKey))) + .forEach(obsUnit -> { + obsUnit.getBrAPIObject() + .setStudyDbId(study.getStudyDbId()); + obsUnit.getBrAPIObject() + .setTrialDbId(study.getTrialDbId()); + }); + } + + private void updateGermplasmDbId(BrAPIGermplasm germplasm) { + this.observationUnitByNameNoScope.values() + .stream() + .filter(obsUnit -> germplasm.getAccessionNumber() != null && + germplasm.getAccessionNumber().equals(obsUnit + .getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.GID).getAsString())) + .forEach(obsUnit -> obsUnit.getBrAPIObject() + .setGermplasmDbId(germplasm.getGermplasmDbId())); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java index 6736a7c28..5ff7a2729 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java @@ -302,13 +302,21 @@ public List commitNewPendingStudiessToBrAPIStore(ImportContext conte updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); List createdStudies = brAPIStudyDAO.createBrAPIStudies(newStudies, program.getId(), upload); + // set the DbId to the for each newly created study + for (BrAPIStudy createdStudy : createdStudies) { + String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData(createdStudy.getStudyName(), program.getKey()); + this.studyByNameNoScope.get(createdStudy_name_no_key) + .getBrAPIObject() + .setStudyDbId(createdStudy.getStudyDbId()); + } + return createdStudies; } // TODO: used by both workflows public List commitUpdatedPendingStudiesToBrAPIStore(ImportContext importContext, PendingData pendingData) { List updatedStudies = new ArrayList<>(); - + return updatedStudies; } } From 70d268cd18101bdc7a681f893ffcd562df1c0afb Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 6 May 2024 17:57:00 -0400 Subject: [PATCH 028/203] add methods for committing pending observations --- .../services/ObservationService.java | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java index d57eca250..dab65dd5c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java @@ -2,26 +2,35 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; +import io.micronaut.http.server.exceptions.InternalServerException; import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; import org.breedinginsight.model.User; +import org.breedinginsight.utilities.Utilities; import tech.tablesaw.columns.Column; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; public class ObservationService { // TODO: used with expUnit workflow @@ -166,4 +175,60 @@ private void updateObservationDependencyValues(Program program) { } } } + + // TODO: used by both workflows + public List commitNewPendingObservationsToBrAPIStore(ImportContext context, PendingData pendingData) { + // filter out observations with no 'value' so they will not be saved + List newObservations = ProcessorData.getNewObjects(this.observationByHash) + .stream() + .filter(obs -> !obs.getValue().isBlank()) + .collect(Collectors.toList()); + + updateObservationDependencyValues(program); + return brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); + + } + + // TODO: used by both workflows + public List commitUpdatedPendingObservationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedObservations = new ArrayList<>(); + Map mutatedObservationByDbId = ProcessorData + .getMutationsByObjectId(observationByHash, BrAPIObservation::getObservationDbId); + + for (Map.Entry entry : mutatedObservationByDbId.entrySet()) { + String id = entry.getKey(); + BrAPIObservation observation = entry.getValue(); + try { + if (observation == null) { + throw new Exception("Null observation"); + } + BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); + updatedObservations.add(updatedObs); + + if (updatedObs == null) { + throw new Exception("Null updated observation"); + } + + if (!Objects.equals(observation.getValue(), updatedObs.getValue()) + || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { + String message; + if (!Objects.equals(observation.getValue(), updatedObs.getValue())) { + message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); + } else { + message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); + } + throw new Exception(message); + } + + } catch (ApiException e) { + log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating observation: ", e); + throw new InternalServerException(e.getMessage(), e); + } + } + + return updatedObservations; + } } From 8fc2f31a33698340a15d42dd8c80da7561c76684 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 7 May 2024 11:33:56 -0400 Subject: [PATCH 029/203] Added bean mapping for factory producing appropriate workflow --- .../migration/V1.22.0__add_experiment_workflows.sql | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql index a78947c38..d5c4b9556 100644 --- a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql +++ b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql @@ -19,7 +19,8 @@ CREATE TABLE importer_mapping_workflow ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, mapping_id UUID NOT NULL, - name TEXT + name TEXT NOT NULL, + bean TEXT NOT NULL ); ALTER TABLE importer_mapping_workflow @@ -32,10 +33,10 @@ DECLARE BEGIN exp_mapping_id := (SELECT id FROM importer_mapping WHERE name = 'ExperimentsTemplateMap'); -INSERT INTO public.importer_mapping_workflow (mapping_id, name) +INSERT INTO public.importer_mapping_workflow (mapping_id, name, bean) VALUES - (exp_mapping_id, 'Create new experiment'), - (exp_mapping_id, 'Append experimental dataset'), - (exp_mapping_id, 'Create new experimental environment'); + (exp_mapping_id, 'Create new experiment', 'CreateNewExperimentWorkflow'), + (exp_mapping_id, 'Append experimental dataset', 'AppendOverwritePhenotypesWorkflow'), + (exp_mapping_id, 'Create new experimental environment', 'CreateNewEnvironmentWorkflow'); END $$; \ No newline at end of file From 4b61708eaa9ece75443f6608b20bdf389b165c93 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 7 May 2024 11:36:12 -0400 Subject: [PATCH 030/203] Added endpoint for retrieving workflows for given mapping id --- .../controllers/ImportController.java | 18 +++++++ .../daos/ImportMappingWorkflowDAO.java | 46 ++++++++++++++++ .../model/workflow/ImportMappingWorkflow.java | 52 +++++++++++++++++++ .../importer/services/FileImportService.java | 14 ++++- 4 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index e0a61117a..5c9ac0ca1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -37,6 +37,7 @@ import org.breedinginsight.api.model.v1.response.metadata.StatusCode; import org.breedinginsight.api.v1.controller.metadata.AddMetadata; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; +import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; import org.breedinginsight.brapps.importer.services.ImportConfigManager; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; import org.breedinginsight.brapps.importer.services.FileImportService; @@ -208,4 +209,21 @@ public HttpResponse>> getSystemMappings(@Nu Response> response = new Response(metadata, new DataResponse<>(result)); return HttpResponse.ok(response); } + + @Get("/import/mappings/{mappingId}/workflows") + @Produces(MediaType.APPLICATION_JSON) + @AddMetadata + @Secured(SecurityRule.IS_ANONYMOUS) + public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { + + List workflows = fileImportService.getWorkflowsForSystemMapping(mappingId); + + List metadataStatus = new ArrayList<>(); + metadataStatus.add(new Status(StatusCode.INFO, "Successful Query")); + Pagination pagination = new Pagination(workflows.size(), workflows.size(), 1, 0); + Metadata metadata = new Metadata(pagination, metadataStatus); + + Response> response = new Response(metadata, new DataResponse<>(workflows)); + return HttpResponse.ok(response); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java new file mode 100644 index 000000000..e6a0a7104 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java @@ -0,0 +1,46 @@ +package org.breedinginsight.brapps.importer.daos; + +import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; +import org.breedinginsight.dao.db.tables.daos.ImporterMappingWorkflowDao; +import org.jooq.Configuration; +import org.jooq.DSLContext; +import org.jooq.Record; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static org.breedinginsight.dao.db.Tables.*; + +public class ImportMappingWorkflowDAO extends ImporterMappingWorkflowDao { + + private DSLContext dsl; + + @Inject + public ImportMappingWorkflowDAO(Configuration config, DSLContext dsl) { + super(config); + this.dsl = dsl; + } + + /** + * Retrieves a list of ImportMappingWorkflow objects associated with the given mappingId. + * + * @param mappingId The UUID of the mapping to retrieve the workflows for. + * @return A list of ImportMappingWorkflow objects. + */ + public List getWorkflowsByImportMappingId(UUID mappingId) { + + List records = dsl.select() + .from(IMPORTER_MAPPING_WORKFLOW) + .where(IMPORTER_MAPPING_WORKFLOW.MAPPING_ID.eq(mappingId)) + .fetch(); + + List workflows = new ArrayList<>(); + for (Record record: records) { + workflows.add(ImportMappingWorkflow.parseSQLRecord(record)); + } + return workflows; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java new file mode 100644 index 000000000..51a70bb5a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java @@ -0,0 +1,52 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; +import lombok.experimental.SuperBuilder; + +import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; +import org.jooq.Record; + +import static org.breedinginsight.dao.db.tables.ImporterMappingWorkflowTable.IMPORTER_MAPPING_WORKFLOW; + +@Getter +@Setter +@Accessors(chain=true) +@ToString +@NoArgsConstructor +@SuperBuilder() +public class ImportMappingWorkflow extends ImporterMappingWorkflowEntity { + + public ImportMappingWorkflow(ImporterMappingWorkflowEntity importMappingWorkflowEntity) { + this.setId(importMappingWorkflowEntity.getId()); + this.setName(importMappingWorkflowEntity.getName()); + } + + public static ImportMappingWorkflow parseSQLRecord(Record record) { + + return ImportMappingWorkflow.builder() + .id(record.getValue(IMPORTER_MAPPING_WORKFLOW.ID)) + .name(record.getValue(IMPORTER_MAPPING_WORKFLOW.NAME)) + .build(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 05c48601d..950465459 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.micronaut.http.HttpStatus; +import io.micronaut.http.annotation.PathVariable; import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.multipart.CompletedFileUpload; import io.micronaut.http.server.exceptions.InternalServerException; @@ -32,6 +33,7 @@ import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.daos.ImportMappingProgramDAO; +import org.breedinginsight.brapps.importer.daos.ImportMappingWorkflowDAO; import org.breedinginsight.brapps.importer.model.ImportProgress; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; @@ -40,8 +42,10 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; +import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingProgramEntity; +import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import org.breedinginsight.services.ProgramService; @@ -80,12 +84,13 @@ public class FileImportService { private final ImportDAO importDAO; private final DSLContext dsl; private final ImportMappingProgramDAO importMappingProgramDAO; + private final ImportMappingWorkflowDAO importMappingWorkflowDAO; @Inject FileImportService(ProgramUserService programUserService, ProgramService programService, MimeTypeParser mimeTypeParser, ImportMappingDAO importMappingDAO, ObjectMapper objectMapper, MappingManager mappingManager, ImportConfigManager configManager, ImportDAO importDAO, DSLContext dsl, ImportMappingProgramDAO importMappingProgramDAO, - UserService userService) { + ImportMappingWorkflowDAO importMappingWorkflowDAO, UserService userService) { this.programUserService = programUserService; this.programService = programService; this.mimeTypeParser = mimeTypeParser; @@ -97,6 +102,7 @@ public class FileImportService { this.dsl = dsl; this.importMappingProgramDAO = importMappingProgramDAO; this.userService = userService; + this.importMappingWorkflowDAO = importMappingWorkflowDAO; } public List getAllImportTypeConfigs() { @@ -559,4 +565,10 @@ public List getSystemMappingByName(String name) { List importMappings = importMappingDAO.getSystemMappingByName(name); return importMappings; } + + public List getWorkflowsForSystemMapping(UUID mappingId) { + return importMappingWorkflowDAO.getWorkflowsByImportMappingId(mappingId); + } + + } From 585947eaaf460c99ea5c6866ec8052a3d4e0424f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 7 May 2024 12:12:20 -0400 Subject: [PATCH 031/203] fix up middleware for refID validation --- .../experiment/ExperimentUtilities.java | 34 +++++++++++++++++++ .../AppendOverwritePhenotypesWorkflow.java | 9 ++--- .../ExpUnitContextService.java | 6 ++-- .../middleware/GetExistingBrAPIData.java | 5 ++- .../middleware/ValidateAllRowsHaveIDs.java | 33 ++++-------------- .../CreateBrAPITrials.java | 2 +- ...ts.java => ExpImportProcessConstants.java} | 6 ++-- .../model/ExpUnitMiddlewareContext.java | 5 ++- .../experiment/model/ProcessedData.java | 5 ++- 9 files changed, 57 insertions(+), 48 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{postBrAPIData => commit}/CreateBrAPITrials.java (65%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/{ExpImportProcessErrorConstants.java => ExpImportProcessConstants.java} (78%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 25b87a0e8..d22f9ae2f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -1,15 +1,21 @@ package org.breedinginsight.brapps.importer.services.processors.experiment; import com.google.gson.JsonObject; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; import org.brapi.v2.model.core.BrAPIStudy; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; public class ExperimentUtilities { @@ -59,4 +65,32 @@ public void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study, Stri additionalInfo.addProperty(BrAPIAdditionalInfoFields.ENV_YEAR, year); } } + + public static Set collateReferenceOUIds(ExpUnitMiddlewareContext context) { + Set referenceOUIds = new HashSet<>(); + boolean hasNoReferenceUnitIds = true; + boolean hasAllReferenceUnitIds = true; + for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + + // Check if ObsUnitID is blank + if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { + hasAllReferenceUnitIds = false; + } else if (referenceOUIds.contains(importRow.getObsUnitID())) { + + // Throw exception if ObsUnitID is repeated + throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); + } else { + // Add ObsUnitID to referenceOUIds + referenceOUIds.add(importRow.getObsUnitID()); + hasNoReferenceUnitIds = false; + } + } + if (!hasNoReferenceUnitIds && !hasAllReferenceUnitIds) { + + // can't proceed if the import has a mix of ObsUnitId for some but not all rows + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, ExpImportProcessConstants.ErrMessage.MISSING_OBS_UNIT_ID_ERROR); + } + return referenceOUIds; + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index fe2e1ae3b..d7de40d00 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -17,18 +17,19 @@ public class AppendOverwritePhenotypesWorkflow implements Workflow { ExpUnitMiddleware middleware; + Provider validateAllRowsHaveIDsProvider; Provider getExistingBrAPIDataProvider; @Inject - public AppendOverwritePhenotypesWorkflow(Provider getExistingBrAPIDataProvider) { + public AppendOverwritePhenotypesWorkflow(Provider validateAllRowsHaveIDsProvider, + Provider getExistingBrAPIDataProvider) { this.middleware.link( - new ValidateAllRowsHaveIDs(), + validateAllRowsHaveIDsProvider.get(), getExistingBrAPIDataProvider.get() ); } @Override public ProcessedData process(ImportContext context) { - ExpUnitMiddlewareContext workflowContext = new ExpUnitMiddlewareContext(); - workflowContext.setImportContext(context); + ExpUnitMiddlewareContext workflowContext = ExpUnitMiddlewareContext.builder().importContext(context).build(); this.middleware.process(workflowContext); // TODO: implement diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java index 85a415717..4f2bd4411 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java @@ -1,7 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite; import io.micronaut.context.annotation.Property; -import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; @@ -12,9 +11,8 @@ import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessErrorConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; -import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; @@ -79,7 +77,7 @@ public Map> fetchReferenceObse missingIds.removeAll(fetchedIds); // throw error reporting any reference IDs with no corresponding stored unit in the brapi data store - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessErrorConstants.COMMA_DELIMITER, missingIds)); + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingIds)); } return pendingUnitById; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java index e87583d6b..9d85306d4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java @@ -10,8 +10,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessErrorConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.utilities.Utilities; @@ -96,7 +95,7 @@ private Map> fetchReferenceObs // throw error reporting any reference IDs with no corresponding stored unit in the brapi data store this.compensate(context, new MiddlewareError(() -> { - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessErrorConstants.COMMA_DELIMITER, missingIds)); + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingIds)); })); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java index 45c728453..4a90c4bec 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java @@ -1,6 +1,8 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; +import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; @@ -8,13 +10,12 @@ import java.util.HashSet; import java.util.Set; +@Slf4j public class ValidateAllRowsHaveIDs extends ExpUnitMiddleware { - private boolean hasAllReferenceUnitIds = true; - private boolean hasNoReferenceUnitIds = true; - @Override public boolean process(ExpUnitMiddlewareContext context) { - context.setReferenceOUIds(collateReferenceOUIds(context)); + + context.getExpUnitContext().setReferenceOUIds(ExperimentUtilities.collateReferenceOUIds(context)); return processNext(context); } @@ -27,27 +28,5 @@ public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError erro return compensatePrior(context, error); } - private Set collateReferenceOUIds(ExpUnitMiddlewareContext context) { - Set referenceOUIds = new HashSet<>(); - for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); - - // Check if ObsUnitID is blank - if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { - hasAllReferenceUnitIds = false; - } else if (referenceOUIds.contains(importRow.getObsUnitID())) { - - // Throw exception if ObsUnitID is repeated - this.compensate(context, new MiddlewareError(()->{ - throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); - })); - - } else { - // Add ObsUnitID to referenceOUIds - referenceOUIds.add(importRow.getObsUnitID()); - hasNoReferenceUnitIds = false; - } - } - return referenceOUIds; - } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/postBrAPIData/CreateBrAPITrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/CreateBrAPITrials.java similarity index 65% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/postBrAPIData/CreateBrAPITrials.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/CreateBrAPITrials.java index b84b0bd3e..d6ea6360b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/postBrAPIData/CreateBrAPITrials.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/CreateBrAPITrials.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.postBrAPIData; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; public class CreateBrAPITrials { } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessErrorConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java similarity index 78% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessErrorConstants.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java index 8509d74f6..4da4414a1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessErrorConstants.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java @@ -2,16 +2,16 @@ import com.fasterxml.jackson.annotation.JsonValue; -public class ExpImportProcessErrorConstants { +public class ExpImportProcessConstants { public static final CharSequence COMMA_DELIMITER = ","; - public enum ExpImportProcessErrMessage { + public enum ErrMessage { MISSING_OBS_UNIT_ID_ERROR("Experimental entities are missing ObsUnitIDs"), PREEXISTING_EXPERIMENT_TITLE("Experiment Title already exists"); private String value; - ExpImportProcessErrMessage(String value) { + ErrMessage(String value) { this.value = value; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java index 740707142..1e81ff833 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java @@ -4,10 +4,9 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -@Getter -@Setter +@Data +@Builder public class ExpUnitMiddlewareContext { - private ImportContext importContext; private ExpUnitContext expUnitContext; private PendingData pendingData; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java index 551af741b..49fe30b12 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java @@ -5,11 +5,10 @@ import java.util.Map; -@Getter -@Setter +@Data @Builder @ToString @NoArgsConstructor public class ProcessedData { - Map statistics; + private Map statistics; } From 6d56b6ef8f58a0b8639da9613213ea61f71bc81e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 7 May 2024 14:04:34 -0400 Subject: [PATCH 032/203] add error handler middleware --- .../AppendOverwritePhenotypesWorkflow.java | 10 ++++++---- .../appendoverwrite/middleware/HandleErr.java | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/HandleErr.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index d7de40d00..125b52d14 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -3,6 +3,7 @@ import io.micronaut.context.annotation.Prototype; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.GetExistingBrAPIData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.HandleErr; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ValidateAllRowsHaveIDs; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; @@ -17,15 +18,16 @@ public class AppendOverwritePhenotypesWorkflow implements Workflow { ExpUnitMiddleware middleware; + Provider handleErrProvider; Provider validateAllRowsHaveIDsProvider; Provider getExistingBrAPIDataProvider; @Inject - public AppendOverwritePhenotypesWorkflow(Provider validateAllRowsHaveIDsProvider, + public AppendOverwritePhenotypesWorkflow(Provider handleErrProvider, + Provider validateAllRowsHaveIDsProvider, Provider getExistingBrAPIDataProvider) { - this.middleware.link( + this.middleware.link(handleErrProvider.get(), validateAllRowsHaveIDsProvider.get(), - getExistingBrAPIDataProvider.get() - ); + getExistingBrAPIDataProvider.get()); } @Override public ProcessedData process(ImportContext context) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/HandleErr.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/HandleErr.java new file mode 100644 index 000000000..267643d63 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/HandleErr.java @@ -0,0 +1,20 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; + +@Slf4j +public class HandleErr extends ExpUnitMiddleware { + @Override + public boolean process(ExpUnitMiddlewareContext context) { + return processNext(context); + } + + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // TODO: handle any error here + + return true; + } +} From c1fe8fa55d4b4170cf52c3b82fef2f18f5c5c765 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 7 May 2024 17:25:02 -0400 Subject: [PATCH 033/203] Simplify record mapping --- .../importer/daos/ImportMappingWorkflowDAO.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java index e6a0a7104..c2ad58c23 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java @@ -30,17 +30,10 @@ public ImportMappingWorkflowDAO(Configuration config, DSLContext dsl) { * @return A list of ImportMappingWorkflow objects. */ public List getWorkflowsByImportMappingId(UUID mappingId) { - - List records = dsl.select() + return dsl.select() .from(IMPORTER_MAPPING_WORKFLOW) .where(IMPORTER_MAPPING_WORKFLOW.MAPPING_ID.eq(mappingId)) - .fetch(); - - List workflows = new ArrayList<>(); - for (Record record: records) { - workflows.add(ImportMappingWorkflow.parseSQLRecord(record)); - } - return workflows; + .fetch(ImportMappingWorkflow::parseSQLRecord); } } From 69537273b7b124cf3f59f57208781730c93f7d30 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 7 May 2024 20:14:23 -0400 Subject: [PATCH 034/203] add ExistingObsUnit middleware --- .../ExpUnitContextService.java | 97 +++++++++++++++++-- .../middleware/ExpUnitMiddleware.java | 8 ++ .../read/brapi/ExistingBrAPIData.java | 24 +++++ .../read/brapi/ExistingObservationUnits.java | 72 ++++++++++++++ .../steps/GetExistingProcessingStep.java | 5 +- .../{services => service}/DatasetService.java | 2 +- .../GermplasmService.java | 2 +- .../LocationService.java | 5 +- .../ObservationService.java | 6 +- .../ObservationUnitService.java | 3 +- .../StatisticsService.java | 2 +- .../{services => service}/StudyService.java | 5 +- .../{services => service}/TrialService.java | 6 +- .../ValidateService.java | 5 +- 14 files changed, 204 insertions(+), 38 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/DatasetService.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/GermplasmService.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/LocationService.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/ObservationService.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/ObservationUnitService.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/StatisticsService.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/StudyService.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/TrialService.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/ValidateService.java (99%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java index 4f2bd4411..39338aac7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java @@ -6,13 +6,16 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.base.ObservationUnit; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; @@ -29,12 +32,77 @@ public class ExpUnitContextService { public ExpUnitContextService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; } + public List getReferenceUnits(Set expUnitIds, + Program program) throws ApiException { + // Retrieve reference Observation Units based on IDs + return brAPIObservationUnitDAO.getObservationUnitsById(new ArrayList(expUnitIds), + context.getImportContext().getProgram()); + } + public List getReferenceUnits(ExpUnitMiddlewareContext context) throws ApiException { + // Retrieve reference Observation Units based on IDs + return brAPIObservationUnitDAO.getObservationUnitsById( + new ArrayList(context.getExpUnitContext().getReferenceOUIds()), + context.getImportContext().getProgram() + ); + } + + public PendingImportObject constructPIOFromExistingUnit(BrAPIObservationUnit unit) { + final PendingImportObject[] pio = new PendingImportObject[]{null}; + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + // Get external reference for the Observation Unit + Optional unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource); + unitXref.ifPresentOrElse( + xref -> { + pio[0] = new PendingImportObject(ImportObjectState.EXISTING, unit, UUID.fromString(xref.getReferenceId())); + }, + () -> { + + // but throw an error if no unit ID + throw new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID"); + } + ); + return pio[0]; + } + + public Map> mapPendingUnitById(List> pios) { + Map> pendingUnitById = new HashMap<>(); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + for (PendingImportObject pio : pios) { + + // Get external reference for the Observation Unit + Optional xref = Utilities.getExternalReference(pio.getBrAPIObject().getExternalReferences(), deltaBreedOUSource); + pendingUnitById.put(xref.get().getReferenceId(),pio); + } + return pendingUnitById; + } + + public Map> mapPendingUnitByNameNoScope(List> pios, + Program program) { + Map> pendingUnitByNameNoScope = new HashMap<>(); + + for (PendingImportObject pio : pios) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pio.getBrAPIObject().getStudyName(), + context.getImportContext().getProgram().getKey() + ); + String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pio.getBrAPIObject().getObservationUnitName(), + context.getImportContext().getProgram().getKey() + ); + pendingUnitByNameNoScope.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName), pio); + } - public Map> fetchReferenceObservationUnits( - ImportContext importContext, - ExpUnitContext expUnitContext - ) throws ApiException { + return pendingUnitByNameNoScope; + } + public Map> fetchReferenceObservationUnits(ImportContext importContext, + ExpUnitContext expUnitContext) throws ApiException { Map> pendingUnitById = new HashMap<>(); try { // Retrieve reference Observation Units based on IDs @@ -87,10 +155,23 @@ public Map> fetchReferenceObse } } - private Map> mapPendingObservationUnitByName( - ExpUnitContext expUnitContext, - ImportContext importContext - ) { + public List collectMissingOUIds(Set referenceIds, List existingUnits) { + List missingIds = new ArrayList<>(referenceIds); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + Set fetchedIds = existingUnits.stream() + .filter(unit ->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).isPresent()) + .map(unit->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).get().getReferenceId()) + .collect(Collectors.toSet()); + missingIds.removeAll(fetchedIds); + + return missingIds; + } + + public Map> mapPendingObservationUnitByName(ImportContext importContext, + ExpUnitContext expUnitContext) { Map> pendingUnitByName = new HashMap<>(); for (Map.Entry> entry : expUnitContext.getPendingObsUnitByOUId().entrySet()) { String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java index 81e4e1a0d..8a6bbcb4f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java @@ -2,7 +2,15 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.Middleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; public abstract class ExpUnitMiddleware extends Middleware { + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // tag an error if it occurred in this local transaction + error.tag(this.getClass().getName()); + // undo the prior local transaction + return compensatePrior(context, error); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java new file mode 100644 index 000000000..4c0a449e8 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java @@ -0,0 +1,24 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +import javax.inject.Inject; +import javax.inject.Provider; + +@Slf4j +public class ExistingBrAPIData extends ExpUnitMiddleware { + ExpUnitMiddleware middleware; + Provider existingObservationUnitsProvider; + + @Inject + public ExistingBrAPIData(Provider existingObservationUnitsProvider) { + this.middleware.link(existingObservationUnitsProvider.get()); + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + return this.middleware.process(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java new file mode 100644 index 000000000..5e48a062b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java @@ -0,0 +1,72 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.ExpUnitContextService; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.model.Program; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class ExistingObservationUnits extends ExpUnitMiddleware { + ExpUnitContextService expUnitContextService; + @Inject + public ExistingObservationUnits(ExpUnitContextService expUnitContextService) { + this.expUnitContextService = expUnitContextService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Set referenceIds; + List missingIds; + List existingUnits; + List> pendingExistingUnits; + Map> pendingUnitById; + Map> pendingUnitByNameNoScope; + + program = context.getImportContext().getProgram(); + try { + // Collect deltabreed-generated exp unit ids listed in the import + referenceIds = context.getExpUnitContext().getReferenceOUIds(); + + // For each id fetch the observation unit from the brapi data store + existingUnits = expUnitContextService.getReferenceUnits(new HashSet<>(referenceIds), program); + if (existingUnits.size() != referenceIds.size()) { + + // Handle case of missing Observation Units in data store + missingIds = expUnitContextService.collectMissingOUIds(new HashSet<>(referenceIds), new ArrayList<>(existingUnits)); + this.compensate(context, new MiddlewareError(() -> { + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingIds)); + })); + } + + // Construct pending import objects from the units + pendingExistingUnits = existingUnits.stream().map(expUnitContextService::constructPIOFromExistingUnit).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending unit by Id + pendingUnitById = expUnitContextService.mapPendingUnitById(new ArrayList<>(pendingExistingUnits)); + + // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed + pendingUnitByNameNoScope = expUnitContextService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingExistingUnits), program); + + // add maps to the context for use in processing import + context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); + context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java index d01324e4f..c4e620620 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -3,7 +3,6 @@ import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.server.exceptions.InternalServerException; -import io.reactivex.functions.Function; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; @@ -22,8 +21,8 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.StudyService; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.TrialService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.services.ProgramLocationService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index 2a7cd2897..e8c37983d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.http.server.exceptions.InternalServerException; import org.brapi.client.v2.model.exceptions.ApiException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/GermplasmService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/GermplasmService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java index c15e71fd9..386b96125 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/GermplasmService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.http.server.exceptions.InternalServerException; import org.apache.commons.lang3.StringUtils; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java index 29e2c851d..624d02842 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java @@ -1,13 +1,10 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.request.BrAPIListNewRequest; -import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index dab65dd5c..f3ac20000 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import com.google.gson.JsonArray; import com.google.gson.JsonObject; @@ -7,11 +7,8 @@ import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservation; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.ProcessorData; @@ -20,7 +17,6 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; -import org.breedinginsight.model.User; import org.breedinginsight.utilities.Utilities; import tech.tablesaw.columns.Column; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index 8dbcbc2d2..da879a2ba 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; @@ -15,7 +15,6 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; -import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StatisticsService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StatisticsService.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StatisticsService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StatisticsService.java index 26e6d7f17..8b9844cdb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StatisticsService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StatisticsService.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import org.apache.commons.lang3.StringUtils; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index 5ff7a2729..2356855ba 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -1,8 +1,6 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.context.annotation.Property; -import io.micronaut.context.annotation.Prototype; -import io.micronaut.http.server.exceptions.InternalServerException; import io.reactivex.functions.Function; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -15,7 +13,6 @@ import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapps.importer.model.imports.PendingImport; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index 57ec2d52b..37de0d895 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; @@ -9,7 +9,6 @@ import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -21,15 +20,12 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; -import java.math.BigInteger; import java.util.*; -import java.util.function.Supplier; import java.util.stream.Collectors; @Singleton diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java index bb0db1662..dd7bc0af2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java @@ -1,10 +1,9 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.api.model.v1.response.ValidationErrors; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -14,13 +13,11 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; -import org.breedinginsight.model.User; import org.breedinginsight.utilities.Utilities; import tech.tablesaw.columns.Column; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; public class ValidateService { From 255eb25ab3948ecf5fcd59a9312fff25739b1c1e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 8 May 2024 10:23:47 -0400 Subject: [PATCH 035/203] refactor unit service methods and document --- .../ExpUnitContextService.java | 88 +---------- .../read/brapi/ExistingBrAPIData.java | 1 + .../read/brapi/ExistingObservationUnits.java | 32 ++-- .../service/ObservationUnitService.java | 140 ++++++++++++++++++ 4 files changed, 161 insertions(+), 100 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java index 39338aac7..fed70bb51 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java @@ -6,15 +6,10 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.model.base.ObservationUnit; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -35,15 +30,7 @@ public ExpUnitContextService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { public List getReferenceUnits(Set expUnitIds, Program program) throws ApiException { // Retrieve reference Observation Units based on IDs - return brAPIObservationUnitDAO.getObservationUnitsById(new ArrayList(expUnitIds), - context.getImportContext().getProgram()); - } - public List getReferenceUnits(ExpUnitMiddlewareContext context) throws ApiException { - // Retrieve reference Observation Units based on IDs - return brAPIObservationUnitDAO.getObservationUnitsById( - new ArrayList(context.getExpUnitContext().getReferenceOUIds()), - context.getImportContext().getProgram() - ); + return brAPIObservationUnitDAO.getObservationUnitsById(new ArrayList(expUnitIds), program); } public PendingImportObject constructPIOFromExistingUnit(BrAPIObservationUnit unit) { @@ -90,70 +77,18 @@ public Map> mapPendingUnitByNa for (PendingImportObject pio : pios) { String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( pio.getBrAPIObject().getStudyName(), - context.getImportContext().getProgram().getKey() + program.getKey() ); String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( pio.getBrAPIObject().getObservationUnitName(), - context.getImportContext().getProgram().getKey() + program.getKey() ); pendingUnitByNameNoScope.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName), pio); } return pendingUnitByNameNoScope; } - public Map> fetchReferenceObservationUnits(ImportContext importContext, - ExpUnitContext expUnitContext) throws ApiException { - Map> pendingUnitById = new HashMap<>(); - try { - // Retrieve reference Observation Units based on IDs - List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( - new ArrayList(expUnitContext.getReferenceOUIds()), - importContext.getProgram() - ); - // Construct the DeltaBreed observation unit source for external references - String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - - if (referenceObsUnits.size() == expUnitContext.getReferenceOUIds().size()) { - for (BrAPIObservationUnit unit : referenceObsUnits) {// Iterate through reference Observation Units - - // Get external reference for the Observation Unit - Optional unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource); - unitXref.ifPresentOrElse( - xref -> { - - // Set pending Observation Unit by its ID - pendingUnitById.put( - xref.getReferenceId(), - new PendingImportObject<>( - ImportObjectState.EXISTING, unit, UUID.fromString(xref.getReferenceId())) - ); - }, - () -> { - - // but throw an error if no unit ID - throw new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID"); - } - ); - } - } else {// Handle case of missing Observation Units in data store - List missingIds = new ArrayList<>(expUnitContext.getReferenceOUIds()); - Set fetchedIds = referenceObsUnits.stream() - .filter(unit ->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).isPresent()) - .map(unit->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).get().getReferenceId()) - .collect(Collectors.toSet()); - missingIds.removeAll(fetchedIds); - - // throw error reporting any reference IDs with no corresponding stored unit in the brapi data store - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingIds)); - } - - return pendingUnitById; - } catch (ApiException e) { - log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new ApiException(e); - } - } public List collectMissingOUIds(Set referenceIds, List existingUnits) { List missingIds = new ArrayList<>(referenceIds); @@ -169,21 +104,4 @@ public List collectMissingOUIds(Set referenceIds, List> mapPendingObservationUnitByName(ImportContext importContext, - ExpUnitContext expUnitContext) { - Map> pendingUnitByName = new HashMap<>(); - for (Map.Entry> entry : expUnitContext.getPendingObsUnitByOUId().entrySet()) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( - entry.getValue().getBrAPIObject().getStudyName(), - importContext.getProgram().getKey() - ); - String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( - entry.getValue().getBrAPIObject().getObservationUnitName(), - importContext.getProgram().getKey() - ); - pendingUnitByName.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName), entry.getValue()); - } - return pendingUnitByName; - } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java index 4c0a449e8..f3c342438 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java @@ -19,6 +19,7 @@ public ExistingBrAPIData(Provider existingObservationU @Override public boolean process(ExpUnitMiddlewareContext context) { + log.debug("reading required BrAPI data from BrAPI service"); return this.middleware.process(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java index 5e48a062b..6255125c9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java @@ -4,11 +4,11 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.ExpUnitContextService; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; import org.breedinginsight.model.Program; import javax.inject.Inject; @@ -17,46 +17,48 @@ @Slf4j public class ExistingObservationUnits extends ExpUnitMiddleware { - ExpUnitContextService expUnitContextService; + ObservationUnitService observationUnitService; + @Inject - public ExistingObservationUnits(ExpUnitContextService expUnitContextService) { - this.expUnitContextService = expUnitContextService; + public ExistingObservationUnits(ObservationUnitService observationUnitService) { + this.observationUnitService = observationUnitService; } @Override public boolean process(ExpUnitMiddlewareContext context) { Program program; - Set referenceIds; + Set expUnitIds; List missingIds; - List existingUnits; - List> pendingExistingUnits; + List brapiUnits; + List> pendingUnits; Map> pendingUnitById; Map> pendingUnitByNameNoScope; + log.debug("fetching existing exp units from BrAPI service"); program = context.getImportContext().getProgram(); try { // Collect deltabreed-generated exp unit ids listed in the import - referenceIds = context.getExpUnitContext().getReferenceOUIds(); + expUnitIds = context.getExpUnitContext().getReferenceOUIds(); // For each id fetch the observation unit from the brapi data store - existingUnits = expUnitContextService.getReferenceUnits(new HashSet<>(referenceIds), program); - if (existingUnits.size() != referenceIds.size()) { + brapiUnits = observationUnitService.getReferenceUnits(new HashSet<>(expUnitIds), program); + if (brapiUnits.size() != expUnitIds.size()) { // Handle case of missing Observation Units in data store - missingIds = expUnitContextService.collectMissingOUIds(new HashSet<>(referenceIds), new ArrayList<>(existingUnits)); + missingIds = observationUnitService.collectMissingOUIds(new HashSet<>(expUnitIds), new ArrayList<>(brapiUnits)); this.compensate(context, new MiddlewareError(() -> { throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingIds)); })); } // Construct pending import objects from the units - pendingExistingUnits = existingUnits.stream().map(expUnitContextService::constructPIOFromExistingUnit).collect(Collectors.toList()); + pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); - // Construct a hashmap to look up the pending unit by Id - pendingUnitById = expUnitContextService.mapPendingUnitById(new ArrayList<>(pendingExistingUnits)); + // Construct a hashmap to look up the pending unit by ID + pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed - pendingUnitByNameNoScope = expUnitContextService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingExistingUnits), program); + pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); // add maps to the context for use in processing import context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index da879a2ba..d7f3708d5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -1,28 +1,168 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.service; +import io.micronaut.context.annotation.Property; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; +import javax.inject.Inject; import java.util.*; import java.util.stream.Collectors; public class ObservationUnitService { + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public ObservationUnitService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + } + + /** + * Retrieves the reference Observation Units based on the provided set of experimental unit IDs and the associated Program. + * This function queries the database to fetch Observation Units that match the specified IDs within the given Program. + * + * @param expUnitIds A set of unique identifiers representing the Experimental Units for which reference Observation Units are required. + * These IDs serve as filters to determine the relevant Observation Units. + * @param program The Program to which the Observation Units belong. This parameter ensures that the fetched units are specific to the designated Program. + * @return A List of BrAPIObservationUnit objects that correspond to the reference Observation Units matching the provided expUnitIds within the given Program. + * @throws ApiException If an error occurs during the retrieval process, an ApiException is thrown to handle exceptional scenarios. + */ + public List getReferenceUnits(Set expUnitIds, + Program program) throws ApiException { + // Retrieve reference Observation Units based on IDs + return brAPIObservationUnitDAO.getObservationUnitsById(new ArrayList(expUnitIds), program); + } + + /** + * Constructs a PendingImportObject of type BrAPIObservationUnit from a given BrAPIObservationUnit object. + * This function is responsible for constructing an import object that represents an observation unit for the Deltabreed system, + * using a provided BrAPIObservationUnit object from a BrAPI source. + * + * @param unit the BrAPIObservationUnit object to be used for constructing the PendingImportObject + * @return a PendingImportObject of type BrAPIObservationUnit representing the imported observation unit + * @throws IllegalStateException if the external reference for the observation unit does not exist + */ + public PendingImportObject constructPIOFromBrapiUnit(BrAPIObservationUnit unit) { + final PendingImportObject[] pio = new PendingImportObject[]{null}; + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + // Get external reference for the Observation Unit + Optional unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource); + unitXref.ifPresentOrElse( + xref -> { + pio[0] = new PendingImportObject(ImportObjectState.EXISTING, unit, UUID.fromString(xref.getReferenceId())); + }, + () -> { + + // but throw an error if no unit ID + throw new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID"); + } + ); + return pio[0]; + } + + /** + * Maps pending observation units by their reference IDs. + * This function takes a list of pending import objects representing BrAPI observation units + * and constructs a map where the key is the external reference ID of the observation unit + * and the value is the pending import object itself. + * + * @param pios List of pending import objects for BrAPI observation units + * @return A map of pending observation units keyed by their external reference ID + */ + public Map> mapPendingUnitById(List> pios) { + Map> pendingUnitById = new HashMap<>(); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + for (PendingImportObject pio : pios) { + + // Get external reference for the Observation Unit + Optional xref = Utilities.getExternalReference(pio.getBrAPIObject().getExternalReferences(), deltaBreedOUSource); + pendingUnitById.put(xref.get().getReferenceId(),pio); + } + + return pendingUnitById; + } + + /** + * This method takes a list of PendingImportObject objects and a Program object as input + * and maps the PendingImportObject objects by their observation unit name without the program scope. + * + * @param pios A list of PendingImportObject objects to be processed + * @param program The Program object representing the scope to be removed from observation unit names + * @return A Map> mapping observation unit names without the program scope to the corresponding PendingImportObject objects + */ + public Map> mapPendingUnitByNameNoScope(List> pios, + Program program) { + Map> pendingUnitByNameNoScope = new HashMap<>(); + + for (PendingImportObject pio : pios) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pio.getBrAPIObject().getStudyName(), + program.getKey() + ); + String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pio.getBrAPIObject().getObservationUnitName(), + program.getKey() + ); + pendingUnitByNameNoScope.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName), pio); + } + + return pendingUnitByNameNoScope; + } + + /** + * Collects missing Observation Unit IDs from a set of reference IDs and a list of existing Observation Units. + * + * This function takes a Set of reference IDs and a List of existing Observation Units, filters out the Observation Units + * that have external references matching a specific source, and returns a List of missing Observation Unit IDs that are + * present in the reference IDs but not found in the existing Observation Units. + * + * @param referenceIds The Set of reference IDs representing all possible Observation Unit IDs to match against. + * @param existingUnits The List of existing Observation Units to compare against the reference IDs. + * @return A List of Observation Unit IDs that are missing from the existing Observation Units but present in the reference IDs. + */ + public List collectMissingOUIds(Set referenceIds, List existingUnits) { + List missingIds = new ArrayList<>(referenceIds); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + Set fetchedIds = existingUnits.stream() + .filter(unit ->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).isPresent()) + .map(unit->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).get().getReferenceId()) + .collect(Collectors.toSet()); + missingIds.removeAll(fetchedIds); + + return missingIds; + } + // TODO: used by expUnit workflow public PendingImportObject fetchOrCreateObsUnitPIO(ImportContext importContext, PendingData pendingData, From 251d0aed24a19f35786776059de056b6bebef269 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 8 May 2024 14:25:06 -0400 Subject: [PATCH 036/203] get trials for required exp units --- ...gBrAPIData.java => RequiredBrAPIData.java} | 6 +- ...its.java => RequiredObservationUnits.java} | 6 +- .../middleware/read/brapi/RequiredTrials.java | 65 +++++++++ .../experiment/service/TrialService.java | 136 ++++++++++++++++++ 4 files changed, 207 insertions(+), 6 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/{ExistingBrAPIData.java => RequiredBrAPIData.java} (81%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/{ExistingObservationUnits.java => RequiredObservationUnits.java} (95%) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java similarity index 81% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java index f3c342438..539f63ca7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java @@ -8,12 +8,12 @@ import javax.inject.Provider; @Slf4j -public class ExistingBrAPIData extends ExpUnitMiddleware { +public class RequiredBrAPIData extends ExpUnitMiddleware { ExpUnitMiddleware middleware; - Provider existingObservationUnitsProvider; + Provider existingObservationUnitsProvider; @Inject - public ExistingBrAPIData(Provider existingObservationUnitsProvider) { + public RequiredBrAPIData(Provider existingObservationUnitsProvider) { this.middleware.link(existingObservationUnitsProvider.get()); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java similarity index 95% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java index 6255125c9..d2e6dab21 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java @@ -16,11 +16,11 @@ import java.util.stream.Collectors; @Slf4j -public class ExistingObservationUnits extends ExpUnitMiddleware { +public class RequiredObservationUnits extends ExpUnitMiddleware { ObservationUnitService observationUnitService; @Inject - public ExistingObservationUnits(ObservationUnitService observationUnitService) { + public RequiredObservationUnits(ObservationUnitService observationUnitService) { this.observationUnitService = observationUnitService; } @@ -34,7 +34,7 @@ public boolean process(ExpUnitMiddlewareContext context) { Map> pendingUnitById; Map> pendingUnitByNameNoScope; - log.debug("fetching existing exp units from BrAPI service"); + log.debug("fetching required exp units from BrAPI service"); program = context.getImportContext().getProgram(); try { // Collect deltabreed-generated exp unit ids listed in the import diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java new file mode 100644 index 000000000..3761a5a7d --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java @@ -0,0 +1,65 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class RequiredTrials extends ExpUnitMiddleware { + TrialService trialService; + + @Inject + public RequiredTrials(TrialService trialService) { + this.trialService = trialService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Map> pendingUnitByNameNoScope; + Set trialDbIds; + List brAPITrials; + List> pendingTrials; + Map> pendingTrialByNameNoScope; + + program = context.getImportContext().getProgram(); + pendingUnitByNameNoScope = context.getPendingData().getObservationUnitByNameNoScope(); + + // nothing to do if there are no required units + if (pendingUnitByNameNoScope.size() == 0) { + return processNext(context); + } + log.debug("fetching from BrAPI service trials belonging to required units"); + + // Get the dbIds of the trials belonging to the required exp units + trialDbIds = pendingUnitByNameNoScope.values().stream().map(pendingUnit -> trialService.getTrialDbIdBelongingToPendingUnit(pendingUnit, program)).collect(Collectors.toSet()); + + // Get the trials belonging to required exp units + brAPITrials = trialDbIds.stream().map(dbId -> trialService.fetchBrapiTrialsBelongingToUnit(dbId, program)).collect(Collectors.toList()); + + // Construct the pending trials from the BrAPI trials + pendingTrials = brAPITrials.stream().map(trialService::constructPIOFromBrapiTrial).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending trial by trial name with the program key removed + pendingTrialByNameNoScope = pendingTrials.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKey(pio.getBrAPIObject().getTrialName(), program.getKey()), pio -> pio)); + + // Add the map to the context for use in processing import + context.getPendingData().setTrialByNameNoScope(pendingTrialByNameNoScope); + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index 37de0d895..d768178ba 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -45,6 +45,96 @@ public TrialService(BrAPITrialDAO brAPITrialDAO, this.studyService = studyService; } + /** + * Retrieves the TrialDbId belonging to a pending unit based on the provided BrAPI observation unit and program. + * If the TrialDbId is directly assigned to the unit, it is returned. Otherwise, the TrialDbId + * assigned to the study belonging to the unit is retrieved. + * + * @param pendingUnit The pending import object representing the BrAPI observation unit. + * @param program The program associated with the pending import. + * @return The TrialDbId belonging to the pending unit. + * @throws IllegalStateException if TrialDbId and StudyDbId are not set for an existing ObservationUnit. + * @throws InternalServerException if there is an internal server error while fetching the TrialDbId. + */ + public String getTrialDbIdBelongingToPendingUnit(PendingImportObject pendingUnit, + Program program) throws IllegalStateException, InternalServerException { + String trialDbId = null; + + BrAPIObservationUnit brapiUnit = pendingUnit.getBrAPIObject(); + if (StringUtils.isBlank(brapiUnit.getTrialDbId()) && StringUtils.isBlank(brapiUnit.getStudyDbId())) { + throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); + } + + // get the trial directly assigned to the unit + if (StringUtils.isNotBlank(brapiUnit.getTrialDbId())) { + trialDbId = brapiUnit.getTrialDbId(); + } else { + + // or get the trial directly assigned to the study belonging to the unit + String studyDbId = brapiUnit.getStudyDbId(); + try { + trialDbId = getTrialDbIdBelongingToStudy(studyDbId, program); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + return trialDbId; + } + + /** + * Fetches BrAPI trials belonging to the specified trial database IDs and program. + * + * This method retrieves a list of BrAPI trials that are associated with the provided set of trial database IDs + * within the context of the given program. It ensures that all specified trial IDs are found in the result + * to maintain data integrity. + * + * @param trialDbIds A set of trial database IDs indicating the trials to fetch. + * @param program The program object representing the program to which the trials belong. + * @return A list of BrAPITrial objects representing the fetched trials. + * @throws IllegalStateException If not all specified trial database IDs are found in the fetched trials. + * @throws InternalServerException If an error occurs during the API call to fetch the trials. + */ + public List fetchBrapiTrialsBelongingToUnits(Set trialDbIds, Program program) { + try { + List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); + if (trials.size() != trialDbIds.size()) { + List missingIds = new ArrayList<>(trialDbIds); + missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); + throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + + return trials; + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + /** + * Fetches a BrAPI trial belonging to a specific unit based on the provided trialDbId and program. + * + * @param trialDbId The unique identifier of the trial to be fetched. + * @param program The program to which the trial belongs. + * @return The BrAPI trial belonging to the specified unit. + * @throws InternalServerException If an internal server error occurs while processing the request. + * @throws IllegalStateException If the trial with the specified trialDbId is not found. + */ + public BrAPITrial fetchBrapiTrialsBelongingToUnit(String trialDbId, Program program) throws InternalServerException { + try { + List trials = brAPITrialDAO.getTrialsByDbIds(Set.of(trialDbId), program); + BrAPITrial trial = trials.get(0); + if (trials.size() == 0) { + throw new IllegalStateException("Trial not found for trialDbId(s): " + trialDbId); + } + + return trial; + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } // TODO: also used in other workflow /** @@ -123,6 +213,30 @@ private Set fetchTrialDbidsForStudies(Set studyDbIds, Program pr return trialDbIds; } + /** + * Retrieves the trialDbId that belongs to a specific study with the given studyDbId and program. + * + * @param studyDbId The unique identifier for the study in the system. + * @param program The program object associated with the study. + * @return The trialDbId associated with the study. + * @throws ApiException if the study with the provided studyDbId is not found in the BrAPI service. + * @throws IllegalStateException if the trialDbId is not set for the existing study. + */ + private String getTrialDbIdBelongingToStudy(String studyDbId, Program program) throws ApiException { + String trialDbId = null; + List studies = studyService.fetchStudiesByDbId(Set.of(studyDbId), program); + if (studies.size() == 0) { + throw new ApiException("Study not found in BrAPI service: " + studyDbId); + } + BrAPIStudy study = studies.get(0); + if (StringUtils.isBlank(study.getTrialDbId())) { + throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); + } + trialDbId = study.getTrialDbId(); + + return trialDbId; + } + /** * This method processes an existing trial, retrieves the experiment ID from the trial's external references, * and caches the trial with the corresponding experiment ID in a map. @@ -149,6 +263,28 @@ private void processAndCacheTrial( new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); } + /** + * Constructs a PendingImportObject containing a BrAPITrial object based on the input BrAPITrial. + * + * This function takes a BrAPITrial object as input and constructs a PendingImportObject which + * encapsulates the trial along with its associated experiment ID. The experiment ID is retrieved + * from the external references of the trial object using utility method getExternalReference. + * If the experiment ID is not found in the external references, an InternalServerException is thrown. + * + * @param trial the BrAPITrial object for which the PendingImportObject is to be constructed + * @return a PendingImportObject containing the BrAPITrial object and its associated experiment ID + * @throws InternalServerException if the experiment ID is not found in the external references of the trial + */ + public PendingImportObject constructPIOFromBrapiTrial(BrAPITrial trial) throws InternalServerException { + PendingImportObject pio = null; + BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(trial.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) + .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); + UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); + pio = new PendingImportObject<>(ImportObjectState.EXISTING, trial, experimentId)); + return pio; + } + /** * Initializes trials by name without scope for the given program. * From 4b8960bbe85f53b6c975d2e1fc624df4a9fafa9f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 8 May 2024 16:16:06 -0400 Subject: [PATCH 037/203] make required studies middleware --- .../read/brapi/RequiredBrAPIData.java | 9 ++- .../read/brapi/RequiredStudies.java | 78 +++++++++++++++++++ .../middleware/read/brapi/RequiredTrials.java | 8 +- .../experiment/service/StudyService.java | 73 +++++++++++++++++ .../experiment/service/TrialService.java | 2 +- 5 files changed, 160 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java index 539f63ca7..aef3d5f9c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java @@ -10,11 +10,14 @@ @Slf4j public class RequiredBrAPIData extends ExpUnitMiddleware { ExpUnitMiddleware middleware; - Provider existingObservationUnitsProvider; + Provider requiredObservationUnitsProvider; + Provider requiredTrialsProvider; @Inject - public RequiredBrAPIData(Provider existingObservationUnitsProvider) { - this.middleware.link(existingObservationUnitsProvider.get()); + public RequiredBrAPIData(Provider requiredObservationUnitsProvider, + Provider requiredTrialsProvider) { + this.middleware.link(requiredObservationUnitsProvider.get(), // Fetch the BrAPI units for the required exp unit ids + requiredTrialsProvider.get()); // Fetch the BrAPI trials belonging to the exp units } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java new file mode 100644 index 000000000..5803d07f0 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java @@ -0,0 +1,78 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +public class RequiredStudies extends ExpUnitMiddleware { + StudyService studyService; + + @Inject + public RequiredStudies(StudyService studyService) { + this.studyService = studyService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Map> pendingUnitByNameNoScope; + Set studyDbIds; + List brAPIStudies; + List> pendingStudies; + Map> pendingStudyByNameNoScope; + + program = context.getImportContext().getProgram(); + pendingUnitByNameNoScope = context.getPendingData().getObservationUnitByNameNoScope(); + + // nothing to do if there are no required units + if (pendingUnitByNameNoScope.size() == 0) { + return processNext(context); + } + log.debug("fetching from BrAPI service studies belonging to required units"); + + // Get the dbIds of the studies belonging to the required exp units + studyDbIds = pendingUnitByNameNoScope.values().stream().map(studyService::getStudyDbIdBelongingToPendingUnit).collect(Collectors.toSet()); + + // Get the BrAPI studies belonging to required exp units + brAPIStudies = studyDbIds.stream().map(dbId -> { + BrAPIStudy study = null; + try { + study = studyService.fetchBrapiStudyByDbId(dbId, program); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + return study; + }).collect(Collectors.toList()); + + // Construct the pending studies from the BrAPI trials + pendingStudies = brAPIStudies.stream().map(pio -> studyService.constructPIOFromBrapiStudy(pio, program)).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending study by study name with the program key removed + pendingStudyByNameNoScope = pendingStudies.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKeyAndUnknownAdditionalData(pio.getBrAPIObject().getStudyName(), program.getKey()), pio -> pio)); + + // Add the map to the context for use in processing import + context.getPendingData().setStudyByNameNoScope(pendingStudyByNameNoScope); + + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java index 3761a5a7d..732c662eb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java @@ -1,15 +1,11 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -48,8 +44,8 @@ public boolean process(ExpUnitMiddlewareContext context) { // Get the dbIds of the trials belonging to the required exp units trialDbIds = pendingUnitByNameNoScope.values().stream().map(pendingUnit -> trialService.getTrialDbIdBelongingToPendingUnit(pendingUnit, program)).collect(Collectors.toSet()); - // Get the trials belonging to required exp units - brAPITrials = trialDbIds.stream().map(dbId -> trialService.fetchBrapiTrialsBelongingToUnit(dbId, program)).collect(Collectors.toList()); + // Get the BrAPI trials belonging to required exp units + brAPITrials = trialDbIds.stream().map(dbId -> trialService.fetchBrapiTrialBelongingToUnit(dbId, program)).collect(Collectors.toList()); // Construct the pending trials from the BrAPI trials pendingTrials = brAPITrials.stream().map(trialService::constructPIOFromBrapiTrial).collect(Collectors.toList()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index 2356855ba..dce3571b6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -77,6 +77,36 @@ public PendingImportObject processAndCacheStudy( return pendingStudy; } + /** + * Constructs a PendingImportObject containing a BrAPIStudy object based on the provided BrAPIStudy and Program. + * This function retrieves the external reference for the study and maps the season dbid to the corresponding year. + * + * @param brAPIStudy The BrAPIStudy object to construct the PendingImportObject from. + * @param program The Program object associated with the study. + * @return A PendingImportObject containing the formatted BrAPIStudy object. + * @throws IllegalStateException If the external reference for the study is not found. + */ + public PendingImportObject constructPIOFromBrapiStudy(BrAPIStudy brAPIStudy, Program program) { + // Retrieve external reference for the study + BrAPIExternalReference xref = Utilities.getExternalReference(brAPIStudy.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) + .orElseThrow(() -> new IllegalStateException("External references weren't found for study (dbid): " + brAPIStudy.getStudyDbId()); + + // Map season dbid to year + String seasonDbId = brAPIStudy.getSeasons().get(0); // It is assumed that the study has only one season + if(StringUtils.isNotBlank(seasonDbId)) { + String seasonYear = seasonDbIdToYear(seasonDbId, program.getId()); + brAPIStudy.setSeasons(Collections.singletonList(seasonYear)); + } + + // Create and return a PendingImportObject for the BrAPIStudy + return new PendingImportObject<>( + ImportObjectState.EXISTING, + (BrAPIStudy) Utilities.formatBrapiObjForDisplay(brAPIStudy, BrAPIStudy.class, program), + UUID.fromString(xref.getReferenceId()) + ); + } + // TODO: used by both workflows private String seasonDbIdToYear(String seasonDbId, UUID programId) { String year = null; @@ -127,6 +157,49 @@ public List fetchStudiesByDbId(Set studyDbIds, Program progr return studies; } + /** + * Fetches a BrAPI study by its unique database identifier and program. + * This method retrieves a BrAPI study identified by the given study database identifier + * and corresponding program. + * + * @param studyDbId A String representing the unique database identifier of the study. + * @param program The Program object specifying the program to which the study belongs. + * @return The BrAPIStudy object representing the study fetched from the database. + * @throws ApiException if the study with the provided studyDbId is not found. + */ + public BrAPIStudy fetchBrapiStudyByDbId(String studyDbId, Program program) throws ApiException { + BrAPIStudy study = null; // Initializing the study object + List studies = brAPIStudyDAO.getStudiesByStudyDbId(Set.of(studyDbId), program); // Retrieving studies from the database + + if (studies.size() == 0) { + throw new IllegalStateException( + "Study not found for studyDbId: " + studyDbId); // Throwing exception if no study is found for provided studyDbId + } + + return studies.get(0); // Returning the first study from the list + } + + /** + * Retrieves the study database ID belonging to a pending unit in BrAPI format. + * + * This method takes a PendingImportObject containing a BrAPIObservationUnit + * object and returns the study database ID associated with the unit, if it exists. + * + * @param pio The PendingImportObject containing the BrAPIObservationUnit object for which the study database ID is to be retrieved. + * @return The study database ID belonging to the pending unit, or null if the unit does not exist or if the study database ID is not set. + */ + public String getStudyDbIdBelongingToPendingUnit(PendingImportObject pio) { + String studyDbId = null; + + // Check if the BrAPI object in the PendingImportObject is not null + if (pio.getBrAPIObject() != null) { + // Retrieve the study database ID from the BrAPIObservationUnit object + studyDbId = pio.getBrAPIObject().getStudyDbId(); + } + + return studyDbId; + } + // TODO: used by both workflows private void initializeStudiesForExistingObservationUnits( Program program, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index d768178ba..86ba933b5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -121,7 +121,7 @@ public List fetchBrapiTrialsBelongingToUnits(Set trialDbIds, * @throws InternalServerException If an internal server error occurs while processing the request. * @throws IllegalStateException If the trial with the specified trialDbId is not found. */ - public BrAPITrial fetchBrapiTrialsBelongingToUnit(String trialDbId, Program program) throws InternalServerException { + public BrAPITrial fetchBrapiTrialBelongingToUnit(String trialDbId, Program program) throws InternalServerException { try { List trials = brAPITrialDAO.getTrialsByDbIds(Set.of(trialDbId), program); BrAPITrial trial = trials.get(0); From 71c145eef9940536acf3531c994dd771058dc77f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 8 May 2024 17:54:53 -0400 Subject: [PATCH 038/203] create required locations middleware --- .../read/brapi/RequiredBrAPIData.java | 7 +- .../read/brapi/RequiredLocations.java | 77 +++++++++++++++++++ .../experiment/service/LocationService.java | 45 ++++++++++- 3 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java index aef3d5f9c..83b15dcae 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java @@ -12,12 +12,15 @@ public class RequiredBrAPIData extends ExpUnitMiddleware { ExpUnitMiddleware middleware; Provider requiredObservationUnitsProvider; Provider requiredTrialsProvider; + Provider requiredStudiesProvider; @Inject public RequiredBrAPIData(Provider requiredObservationUnitsProvider, - Provider requiredTrialsProvider) { + Provider requiredTrialsProvider, + Provider requiredStudiesProvider) { this.middleware.link(requiredObservationUnitsProvider.get(), // Fetch the BrAPI units for the required exp unit ids - requiredTrialsProvider.get()); // Fetch the BrAPI trials belonging to the exp units + requiredTrialsProvider.get(), // Fetch the BrAPI trials belonging to the exp units + requiredStudiesProvider.get()); // Fetch the BrAPI studies belonging to the exp units } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java new file mode 100644 index 000000000..49cdded53 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java @@ -0,0 +1,77 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +public class RequiredLocations extends ExpUnitMiddleware { + LocationService locationService; + + @Inject + public RequiredLocations(LocationService locationService) { + this.locationService = locationService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Set locationDbIds; + List brapiLocations; + List> pendingLocations; + Map> pendingStudyByNameNoScope; + Map> pendingLocationByName; + + program = context.getImportContext().getProgram(); + pendingStudyByNameNoScope = context.getPendingData().getStudyByNameNoScope(); + + // nothing to do if there are no required units + if (pendingStudyByNameNoScope.size() == 0) { + return processNext(context); + } + log.debug("fetching from BrAPI service, locations belonging to required units"); + + // Get the dbIds of the studies belonging to the required exp units + locationDbIds = pendingStudyByNameNoScope.values().stream().map(pio -> pio.getBrAPIObject().getLocationDbId()).collect(Collectors.toSet()); + + // Get the locations belonging to required exp units + brapiLocations = locationDbIds.stream().map(dbId -> { + ProgramLocation location = null; + try { + location = locationService.fetchLocationByDbId(dbId, program); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + return location; + }).collect(Collectors.toList()); + + // Construct the pending locations from the BrAPI locations + pendingLocations = brapiLocations.stream().map(locationService::constructPIOFromBrapiLocation).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending location by location name + pendingLocationByName = pendingLocations.stream().collect(Collectors.toMap(pio -> pio.getBrAPIObject().getName(), pio -> pio)); + + // Add the map to the context for use in processing import + context.getPendingData().setLocationByName(pendingLocationByName); + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java index 624d02842..cbc993bb5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java @@ -17,6 +17,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.utilities.Utilities; import javax.inject.Singleton; @@ -26,6 +27,48 @@ @Singleton @Slf4j public class LocationService { + private final ProgramLocationService programLocationService; + public LocationService(ProgramLocationService programLocationService) { + this.programLocationService = programLocationService; + } + + /** + * Fetches a ProgramLocation based on the provided locationDbId within a given program. + * + * @param locationDbId the unique identifier of the location + * @param program the program in which the location is to be fetched + * @return the ProgramLocation associated with the given locationDbId in the program + * @throws ApiException if the program location cannot be found for the specified locationDbId + */ + public ProgramLocation fetchLocationByDbId(String locationDbId, Program program) throws ApiException { + ProgramLocation programLocation = null; // Initializing the ProgramLocation object + + // Retrieving locations based on the locationDbId and the program's ID + List locations = programLocationService.getLocationsByDbId(List.of(locationDbId), program.getId()); + + // If no locations are found, throw an IllegalStateException with an error message + if (locations.size() == 0) { + throw new IllegalStateException( + "Location not found for locationDbId: " + locationDbId); // Throwing exception if no location is found for provided locationDbId + } + + // Set the programLocation to the first location found in the list of locations + programLocation = locations.get(0); + + // Return the fetched ProgramLocation + return programLocation; + } + + /** + * Constructs a PendingImportObject of type ProgramLocation from a given BrAPI ProgramLocation. + * This method creates a new PendingImportObject with the state set to EXISTING and the BrAPI ProgramLocation as the source object. + * + * @param brapiLocation The BrAPI ProgramLocation from which the PendingImportObject should be constructed + * @return PendingImportObject The PendingImportObject created from the BrAPI ProgramLocation + */ + public PendingImportObject constructPIOFromBrapiLocation(ProgramLocation brapiLocation) { + return new PendingImportObject<>(ImportObjectState.EXISTING, brapiLocation, brapiLocation.getId()); + } // used by expunit workflow public Map> initializeLocationByName( @@ -41,7 +84,7 @@ public Map> initializeLocationByNam .getLocationDbId()) .collect(Collectors.toSet()); try { - existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); + existingLocations.addAll(programLocationService.getLocationsByDbId(locationDbIds, program.getId())); } catch (ApiException e) { log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); From 5635dcf00ad552969eb4a342dc9c0d6afb20cb73 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 9 May 2024 14:33:30 -0400 Subject: [PATCH 039/203] add required dataset middleware --- .../AppendOverwritePhenotypesWorkflow.java | 1 + .../read/brapi/RequiredBrAPIData.java | 13 ++- .../read/brapi/RequiredDatasets.java | 80 +++++++++++++++++++ .../experiment/middleware/Middleware.java | 10 ++- .../experiment/service/DatasetService.java | 74 +++++++++++++++++ 5 files changed, 172 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 125b52d14..b97e40536 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -25,6 +25,7 @@ public class AppendOverwritePhenotypesWorkflow implements Workflow { public AppendOverwritePhenotypesWorkflow(Provider handleErrProvider, Provider validateAllRowsHaveIDsProvider, Provider getExistingBrAPIDataProvider) { + this.middleware.link(handleErrProvider.get(), validateAllRowsHaveIDsProvider.get(), getExistingBrAPIDataProvider.get()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java index 83b15dcae..4cfe2a5ca 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java @@ -13,14 +13,21 @@ public class RequiredBrAPIData extends ExpUnitMiddleware { Provider requiredObservationUnitsProvider; Provider requiredTrialsProvider; Provider requiredStudiesProvider; + Provider requiredLocationsProvider; + Provider requiredDatasetsProvider; @Inject public RequiredBrAPIData(Provider requiredObservationUnitsProvider, Provider requiredTrialsProvider, - Provider requiredStudiesProvider) { + Provider requiredStudiesProvider, + Provider requiredLocationsProvider, + Provider requiredDatasetsProvider) { + this.middleware.link(requiredObservationUnitsProvider.get(), // Fetch the BrAPI units for the required exp unit ids - requiredTrialsProvider.get(), // Fetch the BrAPI trials belonging to the exp units - requiredStudiesProvider.get()); // Fetch the BrAPI studies belonging to the exp units + requiredTrialsProvider.get(), // Fetch the BrAPI trials belonging to the exp units + requiredStudiesProvider.get(), // Fetch the BrAPI studies belonging to the exp units + requiredLocationsProvider.get(), // Fetch the BrAPI locations belonging to the exp units + requiredDatasetsProvider.get()); // Fetch the datasets belonging to the exp units } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java new file mode 100644 index 000000000..ccba81b42 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java @@ -0,0 +1,80 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Slf4j +public class RequiredDatasets extends ExpUnitMiddleware { + private final DatasetService datasetService; + + @Inject + public RequiredDatasets(DatasetService datasetService) { + this.datasetService = datasetService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + String datasetId; + BrAPIListDetails dataset = null; + PendingImportObject pendingDataset; + Map> pendingTrialByNameNoScope; + Map> pendingDatasetByName; + + program = context.getImportContext().getProgram(); + pendingTrialByNameNoScope = context.getPendingData().getTrialByNameNoScope(); + + // nothing to do if there are no trials with dataset ids + if (pendingTrialByNameNoScope.size() == 0 || + !pendingTrialByNameNoScope.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + return processNext(context); + } + log.debug("fetching from BrAPI service, datasets belonging to required units"); + + // Get the dbId of the trial belonging to the required exp units + datasetId = pendingTrialByNameNoScope.values().iterator().next().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + + + // Get the dataset belonging to required exp units + try { + dataset = datasetService.fetchDatasetById(datasetId, program); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + // Construct the pending locations from the BrAPI locations + pendingDataset = datasetService.constructPIOFromDataset(dataset, program); + + // Construct a hashmap to look up the pending dataset by dataset name + pendingDatasetByName = new HashMap<>(); + pendingDatasetByName.put(dataset.getListName(), pendingDataset); + + // Add the map to the context for use in processing import + context.getPendingData().setObsVarDatasetByName(pendingDatasetByName); + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java index 6d097722f..cee2a4c15 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java @@ -10,12 +10,12 @@ public abstract class Middleware { /** * Builds chains of middleware objects. */ - public static Middleware link(Middleware first, Middleware... chain) { + public Middleware link(Middleware first, Middleware... chain) { Middleware head = first; for (Middleware nextInChain: chain) { nextInChain.prior = head.getLastLink(); head.getLastLink().next = nextInChain; - head = nextInChain.getLastLink(); + head = nextInChain; } return first; } @@ -38,7 +38,7 @@ protected boolean processNext(T context) { } return next.process(context); } - + /** * Runs the compensating local transaction for the prior local transaction or ends traversing if * we're at the first local transaction of the transaction. @@ -53,4 +53,8 @@ protected boolean compensatePrior(T context, MiddlewareError error) { private Middleware getLastLink() { return this.next == null ? this : this.next.getLastLink(); } + + private Middleware getFirstLink() { + return this.prior == null ? this : this.prior.getFirstLink(); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index e8c37983d..70169478e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -1,13 +1,16 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.service; +import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIListTypes; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -21,10 +24,81 @@ import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; +import javax.inject.Inject; import java.util.*; import java.util.stream.Collectors; public class DatasetService { + private final BrAPIListDAO brAPIListDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public DatasetService(BrAPIListDAO brapiListDAO) { + this.brAPIListDAO = brapiListDAO; + } + /** + * Module: Dataset Utility + * + * This module provides utility functions for interacting with datasets using the BrAPI standards. + * It includes methods for fetching dataset details, creating new datasets, updating existing datasets, etc. + * Usage: This module can be used in various applications where handling BrAPI-compliant datasets is required. + */ + + /** + * Fetches dataset details by dataset ID and program + * + * This function fetches details of a dataset by its ID and associated program from a data source using the BrAPI standards. + * + * @param id The unique identifier of the dataset to fetch + * @param program The program object associated with the dataset + * @return BrAPIListDetails object containing the details of the dataset + * @throws ApiException if there is an issue with fetching the dataset details from the data source + */ + public BrAPIListDetails fetchDatasetById(String id, Program program) throws ApiException { + BrAPIListDetails dataSetDetails = null; + + // Retrieve existing dataset summaries based on program ID and external reference + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(id)); + + // Check if the existing dataset summaries are returned, throw exception if not + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("Existing dataset summary not returned from BrAPI server"); + } + + // Retrieve dataset details using the list DB ID from the existing dataset summary + dataSetDetails = brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult(); + + return dataSetDetails; + } + + /** + * Constructs a PendingImportObject for a BrAPIListDetails dataset. + * This method retrieves the external reference for the dataset from the existing list + * based on a specific reference source. It then creates a PendingImportObject for the dataset + * with the existing list and reference ID. + * + * @param dataset The BrAPIListDetails dataset for which to construct the PendingImportObject + * @param program + * @return A PendingImportObject containing the dataset with the existing list and reference ID + * @throws IllegalStateException if external references weren't found for the list + */ + public PendingImportObject constructPIOFromDataset(BrAPIListDetails dataset, Program program) { + // Get the external reference for the dataset from the existing list + BrAPIExternalReference xref = Utilities.getExternalReference(dataset.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) + .orElseThrow(() -> new IllegalStateException("External references weren't found for list (dbid): " + dataset.getListDbId()); + + // Create a PendingImportObject for the dataset with the existing list and reference ID + return new PendingImportObject(ImportObjectState.EXISTING, dataset, UUID.fromString(xref.getReferenceId())); + } + // TODO: used by expunit worflow public Map> initializeObsVarDatasetForExistingObservationUnits( Map> trialByName, From b0198f040757d81cb945e27a490605b03a0c0e71 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 9 May 2024 16:12:00 -0400 Subject: [PATCH 040/203] create required germplasm middleware --- .../read/brapi/RequiredBrAPIData.java | 7 +- .../read/brapi/RequiredDatasets.java | 9 +-- .../read/brapi/RequiredGermplasm.java | 73 +++++++++++++++++++ .../experiment/middleware/Middleware.java | 2 +- .../experiment/service/GermplasmService.java | 60 +++++++++++++++ 5 files changed, 141 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java index 4cfe2a5ca..2446c9358 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java @@ -15,19 +15,22 @@ public class RequiredBrAPIData extends ExpUnitMiddleware { Provider requiredStudiesProvider; Provider requiredLocationsProvider; Provider requiredDatasetsProvider; + Provider requiredGermplasmProvider; @Inject public RequiredBrAPIData(Provider requiredObservationUnitsProvider, Provider requiredTrialsProvider, Provider requiredStudiesProvider, Provider requiredLocationsProvider, - Provider requiredDatasetsProvider) { + Provider requiredDatasetsProvider, + Provider requiredGermplasmProvider) { this.middleware.link(requiredObservationUnitsProvider.get(), // Fetch the BrAPI units for the required exp unit ids requiredTrialsProvider.get(), // Fetch the BrAPI trials belonging to the exp units requiredStudiesProvider.get(), // Fetch the BrAPI studies belonging to the exp units requiredLocationsProvider.get(), // Fetch the BrAPI locations belonging to the exp units - requiredDatasetsProvider.get()); // Fetch the datasets belonging to the exp units + requiredDatasetsProvider.get(), // Fetch the dataset belonging to the exp units + requiredGermplasmProvider.get()); // Fetch the germplasm belonging to the exp units } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java index ccba81b42..7e6967bc9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java @@ -2,7 +2,6 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; @@ -11,15 +10,11 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; import org.breedinginsight.model.Program; -import org.breedinginsight.model.ProgramLocation; import javax.inject.Inject; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.stream.Collectors; @Slf4j public class RequiredDatasets extends ExpUnitMiddleware { @@ -49,7 +44,7 @@ public boolean process(ExpUnitMiddlewareContext context) { } log.debug("fetching from BrAPI service, datasets belonging to required units"); - // Get the dbId of the trial belonging to the required exp units + // Get the id of the dataset belonging to the required exp units datasetId = pendingTrialByNameNoScope.values().iterator().next().getBrAPIObject() .getAdditionalInfo() .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) @@ -65,7 +60,7 @@ public boolean process(ExpUnitMiddlewareContext context) { })); } - // Construct the pending locations from the BrAPI locations + // Construct the pending dataset from the BrAPI observation variable list pendingDataset = datasetService.constructPIOFromDataset(dataset, program); // Construct a hashmap to look up the pending dataset by dataset name diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java new file mode 100644 index 000000000..fe8dec94c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java @@ -0,0 +1,73 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.GermplasmService; +import org.breedinginsight.model.Program; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class RequiredGermplasm extends ExpUnitMiddleware { + private final GermplasmService germplasmService; + + @Inject + public RequiredGermplasm(GermplasmService germplasmService) { + + this.germplasmService = germplasmService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Set germplasmDbId; + List brapiGermplasm = null; + List> pendingGermplasm; + Map> pendingGermplasmByGID; + Map> pendingUnitByNameNoScope; + + program = context.getImportContext().getProgram(); + pendingUnitByNameNoScope = context.getPendingData().getObservationUnitByNameNoScope(); + + // nothing to do if there are no observation units + if (pendingUnitByNameNoScope.size() == 0) { + return processNext(context); + } + log.debug("fetching from BrAPI service, germplasm belonging to required units"); + + // Get the dbIds of the germplasm belonging to the required exp units + Set germplasmDbIds = pendingUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + + // Get the dataset belonging to required exp units + try { + brapiGermplasm = germplasmService.fetchGermplasmByDbId(new HashSet<>(germplasmDbIds), program); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + // Construct the pending germplasm from the BrAPI locations + pendingGermplasm = brapiGermplasm.stream().map(germplasmService::constructPIOFromBrapiGermplasm).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending germplasm by gid + pendingGermplasmByGID = pendingGermplasm.stream().collect(Collectors.toMap(germplasmService::getGIDFromGermplasmPIO, pio -> pio)); + + // Add the map to the context for use in processing import + context.getPendingData().setExistingGermplasmByGID(pendingGermplasmByGID); + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java index cee2a4c15..108d6cd88 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java @@ -38,7 +38,7 @@ protected boolean processNext(T context) { } return next.process(context); } - + /** * Runs the compensating local transaction for the prior local transaction or ends traversing if * we're at the first local transaction of the transaction. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java index 386b96125..b4e7127cd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.service; +import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; @@ -7,6 +8,7 @@ import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -17,6 +19,64 @@ import java.util.stream.Collectors; public class GermplasmService { + private final BrAPIGermplasmDAO germplasmDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + public GermplasmService(BrAPIGermplasmDAO germplasmDAO) { + this.germplasmDAO = germplasmDAO; + } + + /** + * Retrieves a list of BrAPI Germplasm objects based on the provided set of database IDs and a Program object. + * + * @param dbIds A Set of database IDs (strings) used to filter germplasm data retrieval. + * @param program The Program object representing the program associated with the germplasm data. + * @return A List of BrAPIGermplasm objects that match the provided database IDs and program. + * @throws ApiException If an error occurs during the retrieval process. + */ + public List fetchGermplasmByDbId(Set dbIds, Program program) throws ApiException { + List brapiGermplasm = null; + brapiGermplasm = germplasmDAO.getGermplasmsByDBID(dbIds, program.getId()); + return brapiGermplasm; + } + + /** + * This method constructs a PendingImportObject for a given BrAPI Germplasm. + * It retrieves the External Reference associated with the Germplasm and constructs a PendingImportObject with ImportObjectState set to EXISTING. + * + * @param brapiGermplasm The BrAPI Germplasm object for which the PendingImportObject needs to be constructed + * @return PendingImportObject A PendingImportObject containing the BrAPI Germplasm object and its External Reference + * @throws IllegalStateException if the External Reference for the Germplasm is not found + */ + public PendingImportObject constructPIOFromBrapiGermplasm(BrAPIGermplasm brapiGermplasm) { + // Initialize the PendingImportObject to null + PendingImportObject pio = null; + + // Retrieve the External Reference associated with the Germplasm from the Utilities class + BrAPIExternalReference xref = Utilities.getExternalReference(brapiGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + // Throw an exception if External Reference is not found + .orElseThrow(() -> new IllegalStateException("External references weren't found for germplasm (dbid): " + brapiGermplasm.getGermplasmDbId())); + + // Construct the PendingImportObject with ImportObjectState set to EXISTING and External Reference UUID + pio = new PendingImportObject<>(ImportObjectState.EXISTING, brapiGermplasm, UUID.fromString(xref.getReferenceId())); + + return pio; + } + + /** + * Retrieves the Germplasm ID from a PendingImportObject containing BrAPI Germplasm data. + * This method extracts the Germplasm ID (GID) from the Accession Number of the BrAPI Germplasm object within the PendingImportObject. + * + * @param pio a PendingImportObject that wraps BrAPI Germplasm data + * @return a String representing the Germplasm ID extracted from the Accession Number + */ + public String getGIDFromGermplasmPIO(PendingImportObject pio) { + String gid = null; + gid = pio.getBrAPIObject().getAccessionNumber(); + return gid; + } + // TODO: used by expunit workflow public Map> initializeGermplasmByGIDForExistingObservationUnits( Map> unitByName, From 61c1229b654dbee8455925ba5da4dd91099a6eff Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 9 May 2024 17:11:24 -0400 Subject: [PATCH 041/203] batch location request --- .../experiment/ExperimentUtilities.java | 2 +- .../read/brapi/RequiredLocations.java | 33 ++++++++----------- .../experiment/service/LocationService.java | 32 +++++++++--------- 3 files changed, 31 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index d22f9ae2f..2a71177ed 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -22,7 +22,7 @@ public class ExperimentUtilities { public static final CharSequence COMMA_DELIMITER = ","; public static final String TIMESTAMP_PREFIX = "TS:"; - + public static List importRowsToExperimentObservations(List importRows) { return importRows.stream() .map(trialImport -> (ExperimentObservation) trialImport) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java index 49cdded53..fed9bf455 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java @@ -49,28 +49,23 @@ public boolean process(ExpUnitMiddlewareContext context) { // Get the dbIds of the studies belonging to the required exp units locationDbIds = pendingStudyByNameNoScope.values().stream().map(pio -> pio.getBrAPIObject().getLocationDbId()).collect(Collectors.toSet()); + try { + // Get the locations belonging to required exp units + brapiLocations = locationService.fetchLocationsByDbId(locationDbIds, program); - // Get the locations belonging to required exp units - brapiLocations = locationDbIds.stream().map(dbId -> { - ProgramLocation location = null; - try { - location = locationService.fetchLocationByDbId(dbId, program); - } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); - } - return location; - }).collect(Collectors.toList()); + // Construct the pending locations from the BrAPI locations + pendingLocations = brapiLocations.stream().map(locationService::constructPIOFromBrapiLocation).collect(Collectors.toList()); - // Construct the pending locations from the BrAPI locations - pendingLocations = brapiLocations.stream().map(locationService::constructPIOFromBrapiLocation).collect(Collectors.toList()); + // Construct a hashmap to look up the pending location by location name + pendingLocationByName = pendingLocations.stream().collect(Collectors.toMap(pio -> pio.getBrAPIObject().getName(), pio -> pio)); - // Construct a hashmap to look up the pending location by location name - pendingLocationByName = pendingLocations.stream().collect(Collectors.toMap(pio -> pio.getBrAPIObject().getName(), pio -> pio)); - - // Add the map to the context for use in processing import - context.getPendingData().setLocationByName(pendingLocationByName); + // Add the map to the context for use in processing import + context.getPendingData().setLocationByName(pendingLocationByName); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } return processNext(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java index cbc993bb5..2fe6815ca 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java @@ -24,6 +24,8 @@ import java.util.*; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; + @Singleton @Slf4j public class LocationService { @@ -33,30 +35,28 @@ public LocationService(ProgramLocationService programLocationService) { } /** - * Fetches a ProgramLocation based on the provided locationDbId within a given program. + * Fetches a list of ProgramLocation objects based on the provided locationDbIds and program ID. * - * @param locationDbId the unique identifier of the location - * @param program the program in which the location is to be fetched - * @return the ProgramLocation associated with the given locationDbId in the program - * @throws ApiException if the program location cannot be found for the specified locationDbId + * @param locationDbIds A set of locationDbIds used to query locations. + * @param program The Program object for which locations are being fetched. + * @return A list of ProgramLocation objects matching the given locationDbIds for the program. + * @throws ApiException if there is an issue with fetching the locations or if any location(s) are not found. */ - public ProgramLocation fetchLocationByDbId(String locationDbId, Program program) throws ApiException { - ProgramLocation programLocation = null; // Initializing the ProgramLocation object + public List fetchLocationsByDbId(Set locationDbIds, Program program) throws ApiException { + List programLocations = null; // Initializing the ProgramLocations list // Retrieving locations based on the locationDbId and the program's ID - List locations = programLocationService.getLocationsByDbId(List.of(locationDbId), program.getId()); + programLocations = programLocationService.getLocationsByDbId(locationDbIds, program.getId()); // If no locations are found, throw an IllegalStateException with an error message - if (locations.size() == 0) { - throw new IllegalStateException( - "Location not found for locationDbId: " + locationDbId); // Throwing exception if no location is found for provided locationDbId + if (locationDbIds.size() != programLocations.size()) { + Set missingIds = new HashSet<>(locationDbIds); + missingIds.removeAll(programLocations.stream().map(ProgramLocation::getLocationDbId).collect(Collectors.toSet())); + throw new IllegalStateException("Location not found for location dbid(s): " + String.join(COMMA_DELIMITER, missingIds)); } - // Set the programLocation to the first location found in the list of locations - programLocation = locations.get(0); - - // Return the fetched ProgramLocation - return programLocation; + // Return the fetched ProgramLocations + return programLocations; } /** From c69762251b376d74714105e8789bb27a226af7f6 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 10 May 2024 10:47:54 -0400 Subject: [PATCH 042/203] batch requests for studies and trials --- .../experiment/ExperimentUtilities.java | 2 +- .../read/brapi/RequiredGermplasm.java | 25 +++++---- .../read/brapi/RequiredObservationUnits.java | 16 ++---- .../read/brapi/RequiredStudies.java | 33 +++++------- .../middleware/read/brapi/RequiredTrials.java | 27 ++++++---- .../service/ObservationUnitService.java | 40 +++++++++++---- .../experiment/service/StudyService.java | 35 +++++++------ .../experiment/service/TrialService.java | 51 ++++++++++++------- 8 files changed, 133 insertions(+), 96 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 2a71177ed..d22f9ae2f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -22,7 +22,7 @@ public class ExperimentUtilities { public static final CharSequence COMMA_DELIMITER = ","; public static final String TIMESTAMP_PREFIX = "TS:"; - + public static List importRowsToExperimentObservations(List importRows) { return importRows.stream() .map(trialImport -> (ExperimentObservation) trialImport) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java index fe8dec94c..1d40ea781 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java @@ -32,7 +32,7 @@ public RequiredGermplasm(GermplasmService germplasmService) { @Override public boolean process(ExpUnitMiddlewareContext context) { Program program; - Set germplasmDbId; + Set germplasmDbIds; List brapiGermplasm = null; List> pendingGermplasm; Map> pendingGermplasmByGID; @@ -48,26 +48,25 @@ public boolean process(ExpUnitMiddlewareContext context) { log.debug("fetching from BrAPI service, germplasm belonging to required units"); // Get the dbIds of the germplasm belonging to the required exp units - Set germplasmDbIds = pendingUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); - - // Get the dataset belonging to required exp units + germplasmDbIds = pendingUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); try { + // Get the dataset belonging to required exp units brapiGermplasm = germplasmService.fetchGermplasmByDbId(new HashSet<>(germplasmDbIds), program); + + // Construct the pending germplasm from the BrAPI locations + pendingGermplasm = brapiGermplasm.stream().map(germplasmService::constructPIOFromBrapiGermplasm).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending germplasm by gid + pendingGermplasmByGID = pendingGermplasm.stream().collect(Collectors.toMap(germplasmService::getGIDFromGermplasmPIO, pio -> pio)); + + // Add the map to the context for use in processing import + context.getPendingData().setExistingGermplasmByGID(pendingGermplasmByGID); } catch (ApiException e) { this.compensate(context, new MiddlewareError(() -> { throw new RuntimeException(e); })); } - // Construct the pending germplasm from the BrAPI locations - pendingGermplasm = brapiGermplasm.stream().map(germplasmService::constructPIOFromBrapiGermplasm).collect(Collectors.toList()); - - // Construct a hashmap to look up the pending germplasm by gid - pendingGermplasmByGID = pendingGermplasm.stream().collect(Collectors.toMap(germplasmService::getGIDFromGermplasmPIO, pio -> pio)); - - // Add the map to the context for use in processing import - context.getPendingData().setExistingGermplasmByGID(pendingGermplasmByGID); - return processNext(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java index d2e6dab21..5a3fa8301 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java @@ -36,20 +36,12 @@ public boolean process(ExpUnitMiddlewareContext context) { log.debug("fetching required exp units from BrAPI service"); program = context.getImportContext().getProgram(); - try { - // Collect deltabreed-generated exp unit ids listed in the import - expUnitIds = context.getExpUnitContext().getReferenceOUIds(); + // Collect deltabreed-generated exp unit ids listed in the import + expUnitIds = context.getExpUnitContext().getReferenceOUIds(); + try { // For each id fetch the observation unit from the brapi data store - brapiUnits = observationUnitService.getReferenceUnits(new HashSet<>(expUnitIds), program); - if (brapiUnits.size() != expUnitIds.size()) { - - // Handle case of missing Observation Units in data store - missingIds = observationUnitService.collectMissingOUIds(new HashSet<>(expUnitIds), new ArrayList<>(brapiUnits)); - this.compensate(context, new MiddlewareError(() -> { - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingIds)); - })); - } + brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); // Construct pending import objects from the units pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java index 5803d07f0..493ae0021 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java @@ -50,28 +50,23 @@ public boolean process(ExpUnitMiddlewareContext context) { // Get the dbIds of the studies belonging to the required exp units studyDbIds = pendingUnitByNameNoScope.values().stream().map(studyService::getStudyDbIdBelongingToPendingUnit).collect(Collectors.toSet()); - // Get the BrAPI studies belonging to required exp units - brAPIStudies = studyDbIds.stream().map(dbId -> { - BrAPIStudy study = null; - try { - study = studyService.fetchBrapiStudyByDbId(dbId, program); - } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); - } - return study; - }).collect(Collectors.toList()); + try { + // Get the BrAPI studies belonging to required exp units + brAPIStudies = studyService.fetchBrapiStudiesByDbId(studyDbIds, program); - // Construct the pending studies from the BrAPI trials - pendingStudies = brAPIStudies.stream().map(pio -> studyService.constructPIOFromBrapiStudy(pio, program)).collect(Collectors.toList()); + // Construct the pending studies from the BrAPI trials + pendingStudies = brAPIStudies.stream().map(pio -> studyService.constructPIOFromBrapiStudy(pio, program)).collect(Collectors.toList()); - // Construct a hashmap to look up the pending study by study name with the program key removed - pendingStudyByNameNoScope = pendingStudies.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKeyAndUnknownAdditionalData(pio.getBrAPIObject().getStudyName(), program.getKey()), pio -> pio)); - - // Add the map to the context for use in processing import - context.getPendingData().setStudyByNameNoScope(pendingStudyByNameNoScope); + // Construct a hashmap to look up the pending study by study name with the program key removed + pendingStudyByNameNoScope = pendingStudies.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKeyAndUnknownAdditionalData(pio.getBrAPIObject().getStudyName(), program.getKey()), pio -> pio)); + // Add the map to the context for use in processing import + context.getPendingData().setStudyByNameNoScope(pendingStudyByNameNoScope); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } return processNext(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java index 732c662eb..aac07d77c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java @@ -1,11 +1,13 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -39,22 +41,29 @@ public boolean process(ExpUnitMiddlewareContext context) { if (pendingUnitByNameNoScope.size() == 0) { return processNext(context); } - log.debug("fetching from BrAPI service trials belonging to required units"); + log.debug("fetching from BrAPI service, trials belonging to required units"); // Get the dbIds of the trials belonging to the required exp units trialDbIds = pendingUnitByNameNoScope.values().stream().map(pendingUnit -> trialService.getTrialDbIdBelongingToPendingUnit(pendingUnit, program)).collect(Collectors.toSet()); + try { + // Get the BrAPI trials belonging to required exp units + brAPITrials = trialService.fetchBrapiTrialsByDbId(trialDbIds, program); - // Get the BrAPI trials belonging to required exp units - brAPITrials = trialDbIds.stream().map(dbId -> trialService.fetchBrapiTrialBelongingToUnit(dbId, program)).collect(Collectors.toList()); + // Construct the pending trials from the BrAPI trials + pendingTrials = brAPITrials.stream().map(trialService::constructPIOFromBrapiTrial).collect(Collectors.toList()); - // Construct the pending trials from the BrAPI trials - pendingTrials = brAPITrials.stream().map(trialService::constructPIOFromBrapiTrial).collect(Collectors.toList()); + // Construct a hashmap to look up the pending trial by trial name with the program key removed + pendingTrialByNameNoScope = pendingTrials.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKey(pio.getBrAPIObject().getTrialName(), program.getKey()), pio -> pio)); + + // Add the map to the context for use in processing import + context.getPendingData().setTrialByNameNoScope(pendingTrialByNameNoScope); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } - // Construct a hashmap to look up the pending trial by trial name with the program key removed - pendingTrialByNameNoScope = pendingTrials.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKey(pio.getBrAPIObject().getTrialName(), program.getKey()), pio -> pio)); - // Add the map to the context for use in processing import - context.getPendingData().setTrialByNameNoScope(pendingTrialByNameNoScope); return processNext(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index d7f3708d5..e19c196d1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -21,6 +21,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; @@ -29,6 +30,8 @@ import java.util.*; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; + public class ObservationUnitService { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; @Property(name = "brapi.server.reference-source") @@ -40,19 +43,36 @@ public ObservationUnitService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { } /** - * Retrieves the reference Observation Units based on the provided set of experimental unit IDs and the associated Program. - * This function queries the database to fetch Observation Units that match the specified IDs within the given Program. + * Retrieves a list of BrAPI (Breeding API) observation units by their database IDs for a given set of experimental unit IDs and program. + * + * This method queries the BrAPIObservationUnitDAO to retrieve BrAPI observation units based on the provided experimental unit IDs and program. + * If the database IDs of the retrieved BrAPI observation units do not match the provided experimental unit IDs, an IllegalStateException is thrown. + * The exception includes information on the missing observation unit database IDs. * - * @param expUnitIds A set of unique identifiers representing the Experimental Units for which reference Observation Units are required. - * These IDs serve as filters to determine the relevant Observation Units. - * @param program The Program to which the Observation Units belong. This parameter ensures that the fetched units are specific to the designated Program. - * @return A List of BrAPIObservationUnit objects that correspond to the reference Observation Units matching the provided expUnitIds within the given Program. - * @throws ApiException If an error occurs during the retrieval process, an ApiException is thrown to handle exceptional scenarios. + * @param expUnitIds a set of experimental unit IDs for which to retrieve BrAPI observation units + * @param program the program for which to retrieve BrAPI observation units + * @return a list of BrAPIObservationUnit objects corresponding to the provided experimental unit IDs + * @throws ApiException if an error occurs during the retrieval of observation units + * @throws IllegalStateException if the retrieved observation units do not match the provided experimental unit IDs */ - public List getReferenceUnits(Set expUnitIds, - Program program) throws ApiException { + public List getObservationUnitsByDbId(Set expUnitIds, Program program) throws ApiException, IllegalStateException { + List brapiUnits = null; + // Retrieve reference Observation Units based on IDs - return brAPIObservationUnitDAO.getObservationUnitsById(new ArrayList(expUnitIds), program); + brapiUnits = brAPIObservationUnitDAO.getObservationUnitsById(expUnitIds, program); + + // If no BrAPI units are found, throw an IllegalStateException with an error message + if (expUnitIds.size() != brapiUnits.size()) { + Set missingIds = new HashSet<>(expUnitIds); + + // Calculate missing IDs based on retrieved BrAPI units + missingIds.removeAll(brapiUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toSet())); + + // Throw exception with missing IDs information + throw new IllegalStateException("Observation unit not found for unit dbid(s): " + String.join(COMMA_DELIMITER, missingIds)); + } + + return brapiUnits; } /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index dce3571b6..baf646689 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -33,6 +33,8 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; + @Singleton @Slf4j public class StudyService { @@ -158,25 +160,28 @@ public List fetchStudiesByDbId(Set studyDbIds, Program progr } /** - * Fetches a BrAPI study by its unique database identifier and program. - * This method retrieves a BrAPI study identified by the given study database identifier - * and corresponding program. + * Fetch BrAPI studies by their database identifiers for a given program. + * + * This method retrieves a list of BrAPI studies based on the provided set of study database identifiers + * and a specified program. It utilizes the BrAPIStudyDAO to fetch studies from the database. * - * @param studyDbId A String representing the unique database identifier of the study. - * @param program The Program object specifying the program to which the study belongs. - * @return The BrAPIStudy object representing the study fetched from the database. - * @throws ApiException if the study with the provided studyDbId is not found. + * @param studyDbIds A set of study database identifiers for filtering the studies. + * @param program The program related to the studies. + * @return A list of BrAPIStudy objects representing the fetched studies. + * @throws ApiException If there are issues in retrieving studies or if any study database identifier is missing. */ - public BrAPIStudy fetchBrapiStudyByDbId(String studyDbId, Program program) throws ApiException { - BrAPIStudy study = null; // Initializing the study object - List studies = brAPIStudyDAO.getStudiesByStudyDbId(Set.of(studyDbId), program); // Retrieving studies from the database - - if (studies.size() == 0) { - throw new IllegalStateException( - "Study not found for studyDbId: " + studyDbId); // Throwing exception if no study is found for provided studyDbId + public List fetchBrapiStudiesByDbId(Set studyDbIds, Program program) throws ApiException { + List brapiStudies = null; // Initializing the study object + brapiStudies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); // Retrieving studies from the database + + // If no studies are found, throw an IllegalStateException with an error message + if (studyDbIds.size() != brapiStudies.size()) { + Set missingIds = new HashSet<>(studyDbIds); + missingIds.removeAll(brapiStudies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toSet())); + throw new IllegalStateException("Study not found for location dbid(s): " + String.join(COMMA_DELIMITER, missingIds)); } - return studies.get(0); // Returning the first study from the list + return brapiStudies; } /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index 86ba933b5..430153d40 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -20,6 +20,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; @@ -28,6 +29,8 @@ import java.util.*; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; + @Singleton @Slf4j public class TrialService { @@ -44,6 +47,12 @@ public TrialService(BrAPITrialDAO brAPITrialDAO, this.brAPITrialDAO = brAPITrialDAO; this.studyService = studyService; } + /** + * Module: BrAPITrialService + * Description: This module contains methods for retrieving BrAPI trials based on trial database IDs and programs. + * brAPITrialDAO: Data Access Object for interacting with BrAPI trials in the database. + * fetchBrapiTrialsByDbId: Method to fetch BrAPI trials based on provided trial database IDs and program. + */ /** * Retrieves the TrialDbId belonging to a pending unit based on the provided BrAPI observation unit and program. @@ -113,28 +122,36 @@ public List fetchBrapiTrialsBelongingToUnits(Set trialDbIds, } /** - * Fetches a BrAPI trial belonging to a specific unit based on the provided trialDbId and program. + * Retrieves a list of BrAPI trials based on a set of trial database IDs and a specified program. * - * @param trialDbId The unique identifier of the trial to be fetched. - * @param program The program to which the trial belongs. - * @return The BrAPI trial belonging to the specified unit. - * @throws InternalServerException If an internal server error occurs while processing the request. - * @throws IllegalStateException If the trial with the specified trialDbId is not found. + * @param trialDbIds a set of trial database IDs used to retrieve the BrAPI trials + * @param program the program associated with the trials + * @return a list of BrAPITrial objects that match the provided trial database IDs and program + * @throws InternalServerException if there is an internal server error during the retrieval process + * @throws ApiException if there is an exception while fetching the trials */ - public BrAPITrial fetchBrapiTrialBelongingToUnit(String trialDbId, Program program) throws InternalServerException { - try { - List trials = brAPITrialDAO.getTrialsByDbIds(Set.of(trialDbId), program); - BrAPITrial trial = trials.get(0); - if (trials.size() == 0) { - throw new IllegalStateException("Trial not found for trialDbId(s): " + trialDbId); - } + public List fetchBrapiTrialsByDbId(Set trialDbIds, Program program) throws InternalServerException, ApiException { + // Initialize the list of BrAPI trials + List brapiTrials = null; - return trial; - } catch (ApiException e) { - log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); + // Retrieve the trials from the DAO based on the provided trial database IDs and program + brapiTrials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); + + // Check if all requested trials were found + if (trialDbIds.size() != brapiTrials.size()) { + // Identify the missing trial database IDs + Set missingIds = new HashSet<>(trialDbIds); + missingIds.removeAll(brapiTrials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toSet())); + + // Throw an exception with the list of missing trial database IDs + throw new IllegalStateException("Trial not found for trial dbid(s): " + String.join(COMMA_DELIMITER, missingIds)); } + + // Return the list of retrieved BrAPI trials + return brapiTrials; } + + // TODO: also used in other workflow /** From 3ca1a30a4ddc76a172d14d2685e242ed4ada7d2b Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 10 May 2024 12:05:09 -0400 Subject: [PATCH 043/203] Pass workflow id through to import services --- .../controllers/UploadController.java | 57 ++++++++++++++++++- .../model/imports/BrAPIImportService.java | 13 +---- .../model/imports/ImportServiceContext.java | 25 ++++++++ .../ExperimentImportService.java | 15 ++++- .../germplasm/GermplasmImportService.java | 11 +++- .../sample/SampleSubmissionImportService.java | 16 +++--- .../importer/services/FileImportService.java | 22 +++++-- 7 files changed, 128 insertions(+), 31 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java index f9d55bd20..053f1dc3c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java @@ -114,7 +114,7 @@ public HttpResponse> commitData(@PathVariable UUID prog @PathVariable UUID uploadId, @Body @Nullable Map userInput) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, actingUser, userInput, true); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, null, actingUser, userInput, true); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { @@ -140,7 +140,60 @@ public HttpResponse> previewData(@PathVariable UUID pro @PathVariable UUID uploadId) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, actingUser, null, false); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, null, actingUser, null, false); + Response response = new Response(result); + return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + return HttpResponse.notFound(); + } catch (AuthorizationException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.FORBIDDEN, e.getMessage()); + } catch (UnprocessableEntityException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } catch (HttpStatusException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(e.getStatus(), e.getMessage()); + } + } + + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/preview") + @Produces(MediaType.APPLICATION_JSON) + @AddMetadata + @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) + public HttpResponse> previewData(@PathVariable UUID programId, @PathVariable UUID mappingId, + @PathVariable UUID workflowId, @PathVariable UUID uploadId) { + try { + AuthenticatedUser actingUser = securityService.getUser(); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, null, false); + Response response = new Response(result); + return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + return HttpResponse.notFound(); + } catch (AuthorizationException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.FORBIDDEN, e.getMessage()); + } catch (UnprocessableEntityException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } catch (HttpStatusException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(e.getStatus(), e.getMessage()); + } + } + + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/commit") + @Produces(MediaType.APPLICATION_JSON) + @AddMetadata + @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) + public HttpResponse> commitData(@PathVariable UUID programId, @PathVariable UUID mappingId, + @PathVariable UUID workflowId, @PathVariable UUID uploadId, + @Body @Nullable Map userInput) { + try { + AuthenticatedUser actingUser = securityService.getUser(); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, userInput, true); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index 1d520371c..d06454c30 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -17,18 +17,7 @@ package org.breedinginsight.brapps.importer.model.imports; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.ValidatorException; -import tech.tablesaw.api.Table; - -import java.util.List; public interface BrAPIImportService { String getImportTypeId(); @@ -48,6 +37,6 @@ default String getMissingUserInputMsg(String fieldName) { default String getWrongUserInputDataTypeMsg(String fieldName, String typeName) { return String.format("User input, \"%s\" must be an %s", fieldName, typeName); } - ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + ImportPreviewResponse process(ImportServiceContext context) throws Exception; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java new file mode 100644 index 000000000..ef7639c2f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -0,0 +1,25 @@ +package org.breedinginsight.brapps.importer.model.imports; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import tech.tablesaw.api.Table; +import java.util.List; +import java.util.UUID; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ImportServiceContext { + private UUID workflowId; + private List brAPIImports; + private Table data; + private Program program; + private ImportUpload upload; + private User user; + private boolean commit; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index cd795564a..965a7d20f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -21,6 +21,7 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; @@ -66,12 +67,22 @@ public String getMissingColumnMsg(String columnName) { } @Override - public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + public ImportPreviewResponse process(ImportServiceContext context) throws Exception { ImportPreviewResponse response = null; List processors = List.of(experimentProcessorProvider.get()); - response = processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); + // TODO: change to calling process directly on processor (not using processor manager and pass along workflowId) + if (context.getWorkflowId() != null) { + log.info("Workflow UUID: " + context.getWorkflowId()); + } + response = processorManagerProvider.get().process(context.getBrAPIImports(), + processors, + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); return response; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index b4eac6b96..ce52f483f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -21,6 +21,7 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.services.processors.GermplasmProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; @@ -62,12 +63,18 @@ public String getImportTypeId() { } @Override - public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + public ImportPreviewResponse process(ImportServiceContext context) throws Exception { ImportPreviewResponse response = null; List processors = List.of(germplasmProcessorProvider.get()); - response = processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); + response = processorManagerProvider.get().process(context.getBrAPIImports(), + processors, + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); return response; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java index 434626e68..2c8e8b7b5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java @@ -21,6 +21,7 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; @@ -59,13 +60,14 @@ public BrAPIImport getImportClass() { } @Override - public ImportPreviewResponse process(List brAPIImports, - Table data, - Program program, - ImportUpload upload, - User user, - Boolean commit) throws Exception { + public ImportPreviewResponse process(ImportServiceContext context) throws Exception { List processors = List.of(sampleProcessorProvider.get()); - return processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); + return processorManagerProvider.get().process(context.getBrAPIImports(), + processors, + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 950465459..960a8077f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -38,6 +38,7 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.response.ImportResponse; @@ -328,7 +329,7 @@ public ImportResponse uploadData(UUID programId, UUID mappingId, AuthenticatedUs return response; } - public ImportResponse updateUpload(UUID programId, UUID uploadId, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws + public ImportResponse updateUpload(UUID programId, UUID uploadId, UUID workflowId, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws DoesNotExistException, UnprocessableEntityException, AuthorizationException { Program program = validateRequest(programId, actingUser); @@ -378,7 +379,7 @@ public ImportResponse updateUpload(UUID programId, UUID uploadId, AuthenticatedU } else { brAPIImportList = mappingManager.map(mappingConfig, data); } - processFile(brAPIImportList, data, program, upload, user, commit, importService, actingUser); + processFile(workflowId, brAPIImportList, data, program, upload, user, commit, importService, actingUser); } catch (UnprocessableEntityException e) { log.error(e.getMessage(), e); ImportProgress progress = upload.getProgress(); @@ -424,13 +425,22 @@ public ImportUpload setDynamicColumns(ImportUpload newUpload, Table data, Import return newUpload; } - private void processFile(List finalBrAPIImportList, Table data, Program program, - ImportUpload upload, User user, Boolean commit, BrAPIImportService importService, - AuthenticatedUser actingUser) { + private void processFile(UUID workflowId, List finalBrAPIImportList, Table data, Program program, + ImportUpload upload, User user, Boolean commit, BrAPIImportService importService, + AuthenticatedUser actingUser) { // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { - importService.process(finalBrAPIImportList, data, program, upload, user, commit); + ImportServiceContext context = ImportServiceContext.builder() + .workflowId(workflowId) + .brAPIImports(finalBrAPIImportList) + .data(data) + .program(program) + .upload(upload) + .user(user) + .commit(commit) + .build(); + importService.process(context); } catch (UnprocessableEntityException e) { log.error(e.getMessage(), e); ImportProgress progress = upload.getProgress(); From 7108089af495dc99ad3e983b6c4cfb89f777d54d Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 10 May 2024 14:58:33 -0400 Subject: [PATCH 044/203] make link method static --- .../AppendOverwritePhenotypesWorkflow.java | 3 +- .../middleware/process/DataValidation.java | 65 +++++++++++++++++++ .../middleware/process/FieldValidation.java | 65 +++++++++++++++++++ .../process/ImportPreviewStatistics.java | 36 ++++++++++ .../process/NewPendingBrAPIObjects.java | 65 +++++++++++++++++++ .../middleware/process/TraitVerification.java | 65 +++++++++++++++++++ .../read/brapi/RequiredBrAPIData.java | 13 ++-- .../read/brapi/RequiredObservationUnits.java | 1 - .../experiment/middleware/Middleware.java | 2 +- 9 files changed, 306 insertions(+), 9 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/DataValidation.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/NewPendingBrAPIObjects.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index b97e40536..2d8664968 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -5,6 +5,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.GetExistingBrAPIData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.HandleErr; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ValidateAllRowsHaveIDs; +import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.Middleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; @@ -26,7 +27,7 @@ public AppendOverwritePhenotypesWorkflow(Provider handleErrProvider, Provider validateAllRowsHaveIDsProvider, Provider getExistingBrAPIDataProvider) { - this.middleware.link(handleErrProvider.get(), + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(handleErrProvider.get(), validateAllRowsHaveIDsProvider.get(), getExistingBrAPIDataProvider.get()); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/DataValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/DataValidation.java new file mode 100644 index 000000000..65b4c7998 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/DataValidation.java @@ -0,0 +1,65 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.model.Program; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class DataValidation extends ExpUnitMiddleware { + ObservationUnitService observationUnitService; + + @Inject + public DataValidation(ObservationUnitService observationUnitService) { + this.observationUnitService = observationUnitService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Set expUnitIds; + List missingIds; + List brapiUnits; + List> pendingUnits; + Map> pendingUnitById; + Map> pendingUnitByNameNoScope; + + log.debug("fetching required exp units from BrAPI service"); + program = context.getImportContext().getProgram(); + + // Collect deltabreed-generated exp unit ids listed in the import + expUnitIds = context.getExpUnitContext().getReferenceOUIds(); + try { + // For each id fetch the observation unit from the brapi data store + brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); + + // Construct pending import objects from the units + pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending unit by ID + pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); + + // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed + pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); + + // add maps to the context for use in processing import + context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); + context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java new file mode 100644 index 000000000..d27190547 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java @@ -0,0 +1,65 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.model.Program; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class FieldValidation extends ExpUnitMiddleware { + ObservationUnitService observationUnitService; + + @Inject + public FieldValidation(ObservationUnitService observationUnitService) { + this.observationUnitService = observationUnitService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Set expUnitIds; + List missingIds; + List brapiUnits; + List> pendingUnits; + Map> pendingUnitById; + Map> pendingUnitByNameNoScope; + + log.debug("fetching required exp units from BrAPI service"); + program = context.getImportContext().getProgram(); + + // Collect deltabreed-generated exp unit ids listed in the import + expUnitIds = context.getExpUnitContext().getReferenceOUIds(); + try { + // For each id fetch the observation unit from the brapi data store + brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); + + // Construct pending import objects from the units + pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending unit by ID + pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); + + // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed + pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); + + // add maps to the context for use in processing import + context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); + context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java new file mode 100644 index 000000000..a2a446f4c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java @@ -0,0 +1,36 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +import javax.inject.Inject; +import javax.inject.Provider; + +@Slf4j +public class ImportPreviewStatistics extends ExpUnitMiddleware { + ExpUnitMiddleware middleware; + private Provider traitVerificationProvider; + private Provider newPendingBrAPIObjectsProvider; + private Provider dataValidationProvider; + private Provider fieldValidationProvider; + + @Inject + public ImportPreviewStatistics(Provider traitVerificationProvider, + Provider newPendingBrAPIObjectsProvider, + Provider dataValidationProvider, + Provider fieldValidationProvider) { + + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( + traitVerificationProvider.get(), // Verify observation variable columns in import belong to program + newPendingBrAPIObjectsProvider.get(), // Construct Pending import objects for new BrAPI data + dataValidationProvider.get(), // Validate data + fieldValidationProvider.get()); // Validate fields + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + log.debug("generating import preview statistics"); + return this.middleware.process(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/NewPendingBrAPIObjects.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/NewPendingBrAPIObjects.java new file mode 100644 index 000000000..fc6b63f98 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/NewPendingBrAPIObjects.java @@ -0,0 +1,65 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.model.Program; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class NewPendingBrAPIObjects extends ExpUnitMiddleware { + ObservationUnitService observationUnitService; + + @Inject + public NewPendingBrAPIObjects(ObservationUnitService observationUnitService) { + this.observationUnitService = observationUnitService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Set expUnitIds; + List missingIds; + List brapiUnits; + List> pendingUnits; + Map> pendingUnitById; + Map> pendingUnitByNameNoScope; + + log.debug("fetching required exp units from BrAPI service"); + program = context.getImportContext().getProgram(); + + // Collect deltabreed-generated exp unit ids listed in the import + expUnitIds = context.getExpUnitContext().getReferenceOUIds(); + try { + // For each id fetch the observation unit from the brapi data store + brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); + + // Construct pending import objects from the units + pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending unit by ID + pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); + + // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed + pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); + + // add maps to the context for use in processing import + context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); + context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java new file mode 100644 index 000000000..facbe014f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java @@ -0,0 +1,65 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.model.Program; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class TraitVerification extends ExpUnitMiddleware { + ObservationUnitService observationUnitService; + + @Inject + public TraitVerification(ObservationUnitService observationUnitService) { + this.observationUnitService = observationUnitService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Set expUnitIds; + List missingIds; + List brapiUnits; + List> pendingUnits; + Map> pendingUnitById; + Map> pendingUnitByNameNoScope; + + log.debug("fetching required exp units from BrAPI service"); + program = context.getImportContext().getProgram(); + + // Collect deltabreed-generated exp unit ids listed in the import + expUnitIds = context.getExpUnitContext().getReferenceOUIds(); + try { + // For each id fetch the observation unit from the brapi data store + brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); + + // Construct pending import objects from the units + pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending unit by ID + pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); + + // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed + pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); + + // add maps to the context for use in processing import + context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); + context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java index 2446c9358..180e6957a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java @@ -25,12 +25,13 @@ public RequiredBrAPIData(Provider requiredObservationU Provider requiredDatasetsProvider, Provider requiredGermplasmProvider) { - this.middleware.link(requiredObservationUnitsProvider.get(), // Fetch the BrAPI units for the required exp unit ids - requiredTrialsProvider.get(), // Fetch the BrAPI trials belonging to the exp units - requiredStudiesProvider.get(), // Fetch the BrAPI studies belonging to the exp units - requiredLocationsProvider.get(), // Fetch the BrAPI locations belonging to the exp units - requiredDatasetsProvider.get(), // Fetch the dataset belonging to the exp units - requiredGermplasmProvider.get()); // Fetch the germplasm belonging to the exp units + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( + requiredObservationUnitsProvider.get(), // Fetch the BrAPI units for the required exp unit ids + requiredTrialsProvider.get(), // Fetch the BrAPI trials belonging to the exp units + requiredStudiesProvider.get(), // Fetch the BrAPI studies belonging to the exp units + requiredLocationsProvider.get(), // Fetch the BrAPI locations belonging to the exp units + requiredDatasetsProvider.get(), // Fetch the dataset belonging to the exp units + requiredGermplasmProvider.get()); // Fetch the germplasm belonging to the exp units } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java index 5a3fa8301..ba19b96f3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java @@ -5,7 +5,6 @@ import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java index 108d6cd88..c3c78cdd6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java @@ -10,7 +10,7 @@ public abstract class Middleware { /** * Builds chains of middleware objects. */ - public Middleware link(Middleware first, Middleware... chain) { + public static Middleware link(Middleware first, Middleware... chain) { Middleware head = first; for (Middleware nextInChain: chain) { nextInChain.prior = head.getLastLink(); From c8e0bfe736957eb5b3ddd5186cd7ab6096580c4a Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 13 May 2024 16:45:25 -0400 Subject: [PATCH 045/203] Add workflow factory to create workflows from ids --- .../daos/ImportMappingWorkflowDAO.java | 12 +++ .../model/imports/ImportServiceContext.java | 21 ++++- .../ExperimentImportService.java | 8 +- .../model/workflow/ImportContext.java | 47 +++++++++++ .../model/workflow/ImportMappingWorkflow.java | 2 - .../model/workflow/ProcessedData.java | 31 +++++++ .../importer/model/workflow/Workflow.java | 36 ++++++++ .../importer/services/FileImportService.java | 14 +++- .../workflow/CreateNewExperimentWorkflow.java | 50 +++++++++++ .../services/workflow/WorkflowFactory.java | 84 +++++++++++++++++++ 10 files changed, 296 insertions(+), 9 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java index c2ad58c23..21aca395a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java @@ -9,6 +9,7 @@ import javax.inject.Inject; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.UUID; import static org.breedinginsight.dao.db.Tables.*; @@ -36,4 +37,15 @@ public List getWorkflowsByImportMappingId(UUID mappingId) .fetch(ImportMappingWorkflow::parseSQLRecord); } + /** + * Retrieves a workflow by its ID. + * + * @param workflowId The ID of the workflow to retrieve. + * @return An Optional containing the ImportMappingWorkflow if found, otherwise an empty Optional. + */ + public Optional getWorkflowById(UUID workflowId) { + return Optional.ofNullable(fetchOneById(workflowId)) + .map(ImportMappingWorkflow::new); + } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java index ef7639c2f..45393f67c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -1,12 +1,29 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.model.imports; import lombok.*; import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import tech.tablesaw.api.Table; import java.util.List; -import java.util.UUID; @Getter @Setter @@ -15,7 +32,7 @@ @AllArgsConstructor @NoArgsConstructor public class ImportServiceContext { - private UUID workflowId; + private Workflow workflow; private List brAPIImports; private Table data; private Program program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 965a7d20f..c6f68b251 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -72,10 +72,12 @@ public ImportPreviewResponse process(ImportServiceContext context) ImportPreviewResponse response = null; List processors = List.of(experimentProcessorProvider.get()); - // TODO: change to calling process directly on processor (not using processor manager and pass along workflowId) - if (context.getWorkflowId() != null) { - log.info("Workflow UUID: " + context.getWorkflowId()); + + if (context.getWorkflow() != null) { + log.info("Workflow: " + context.getWorkflow().getName()); } + + // TODO: change to calling workflow process instead of processor manager response = processorManagerProvider.get().process(context.getBrAPIImports(), processors, context.getData(), diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java new file mode 100644 index 000000000..121ef6b19 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java @@ -0,0 +1,47 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import tech.tablesaw.api.Table; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ImportContext { + private UUID workflowId; + private ImportUpload upload; + private List importRows; + private Map mappedBrAPIImport; + private Table data; + private Program program; + private User user; + private boolean commit; +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java index 51a70bb5a..0a23e1aae 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java @@ -36,12 +36,10 @@ @NoArgsConstructor @SuperBuilder() public class ImportMappingWorkflow extends ImporterMappingWorkflowEntity { - public ImportMappingWorkflow(ImporterMappingWorkflowEntity importMappingWorkflowEntity) { this.setId(importMappingWorkflowEntity.getId()); this.setName(importMappingWorkflowEntity.getName()); } - public static ImportMappingWorkflow parseSQLRecord(Record record) { return ImportMappingWorkflow.builder() diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java new file mode 100644 index 000000000..e73d6ad48 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -0,0 +1,31 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; + +import java.util.Map; + +@Data +@Builder +@ToString +@NoArgsConstructor +public class ProcessedData { + private Map statistics; +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java new file mode 100644 index 000000000..9ea0e7bf9 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -0,0 +1,36 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +public interface Workflow { + + /** + * Processes the given import context and returns the processed data. + * + * @param context the import context containing the necessary data for processing + * @return the processed data + */ + ProcessedData process(ImportContext context); + + /** + * Retrieves the name of the Workflow for logging display purposes. + * + * @return the name of the Workflow + */ + String getName(); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 960a8077f..2c3f7854f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -44,6 +44,8 @@ import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.workflow.WorkflowFactory; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingProgramEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; @@ -86,12 +88,13 @@ public class FileImportService { private final DSLContext dsl; private final ImportMappingProgramDAO importMappingProgramDAO; private final ImportMappingWorkflowDAO importMappingWorkflowDAO; + private final WorkflowFactory workflowFactory; @Inject FileImportService(ProgramUserService programUserService, ProgramService programService, MimeTypeParser mimeTypeParser, ImportMappingDAO importMappingDAO, ObjectMapper objectMapper, MappingManager mappingManager, ImportConfigManager configManager, ImportDAO importDAO, DSLContext dsl, ImportMappingProgramDAO importMappingProgramDAO, - ImportMappingWorkflowDAO importMappingWorkflowDAO, UserService userService) { + ImportMappingWorkflowDAO importMappingWorkflowDAO, WorkflowFactory workflowFactory, UserService userService) { this.programUserService = programUserService; this.programService = programService; this.mimeTypeParser = mimeTypeParser; @@ -104,6 +107,7 @@ public class FileImportService { this.importMappingProgramDAO = importMappingProgramDAO; this.userService = userService; this.importMappingWorkflowDAO = importMappingWorkflowDAO; + this.workflowFactory = workflowFactory; } public List getAllImportTypeConfigs() { @@ -431,8 +435,14 @@ private void processFile(UUID workflowId, List finalBrAPIImportList // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { + Workflow workflow = null; + if (workflowId != null) { + Optional optionalWorkflow = workflowFactory.getWorkflow(workflowId); + workflow = optionalWorkflow.orElse(null); + } + ImportServiceContext context = ImportServiceContext.builder() - .workflowId(workflowId) + .workflow(workflow) .brAPIImports(finalBrAPIImportList) .data(data) .program(program) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java new file mode 100644 index 000000000..6f6ab661a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -0,0 +1,50 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; + +import javax.inject.Named; + +/** + * This class represents a workflow for creating a new experiment. The bean name must match the appropriate bean column + * value in the import_mapping_workflow db table + */ +@Prototype +@Named("CreateNewExperimentWorkflow") +public class CreateNewExperimentWorkflow implements Workflow { + + @Override + public ProcessedData process(ImportContext context) { + // TODO + return null; + } + + /** + * Retrieves the name of the workflow. This is used for logging display purposes. + * + * @return the name of the workflow + */ + @Override + public String getName() { + return "CreateNewExperimentWorkflow"; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java new file mode 100644 index 000000000..60a2fd884 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java @@ -0,0 +1,84 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.workflow; + +import io.micronaut.context.ApplicationContext; +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.exceptions.NoSuchBeanException; +import io.micronaut.inject.qualifiers.Qualifiers; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.daos.ImportMappingWorkflowDAO; +import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; + +import javax.inject.Inject; +import java.util.Optional; +import java.util.UUID; + +@Factory +@Slf4j +public class WorkflowFactory { + + private final ImportMappingWorkflowDAO importMappingWorkflowDAO; + private final ApplicationContext applicationContext; + + @Inject + public WorkflowFactory(ImportMappingWorkflowDAO importMappingWorkflowDAO, + ApplicationContext applicationContext) { + this.importMappingWorkflowDAO = importMappingWorkflowDAO; + this.applicationContext = applicationContext; + } + + /** + * Produces the appropriate workflow instance based on the import context + * + * @param context the import context + * @return an Optional containing the workflow if found, otherwise an empty Optional + * + * @throws IllegalStateException + * @throws NoSuchBeanException + */ + public Optional getWorkflow(UUID workflowId) { + + if (workflowId != null) { + // construct workflow from db record + Optional workflowOptional = importMappingWorkflowDAO.getWorkflowById(workflowId); + if (workflowOptional.isPresent()) { + ImportMappingWorkflow importMappingworkflow = workflowOptional.orElseThrow(() -> { + String msg = "Must have record in db for workflowId"; + log.error(msg); + return new IllegalStateException(msg); + }); + + // newer versions of micronaut have fancier ways to do this using annotations with provider but as + // far as I can tell it's not available in 2.5 + Workflow workflow; + try { + workflow = applicationContext.getBean(Workflow.class, Qualifiers.byName(importMappingworkflow.getBean())); + } catch (NoSuchBeanException e) { + log.error("Could not find workflow class implementation for bean: " + importMappingworkflow.getBean()); + throw e; + } + + return Optional.of(workflow); + } + } + + return Optional.empty(); + } +} From 4391479ee6e2d257e0713c6c6b7cbfc631ee030c Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 13 May 2024 16:50:45 -0400 Subject: [PATCH 046/203] Add missing file header --- .../daos/ImportMappingWorkflowDAO.java | 19 +++++++++++++++++-- .../importer/services/FileImportService.java | 2 +- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java index 21aca395a..216506b5c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java @@ -1,13 +1,28 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.daos; import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; import org.breedinginsight.dao.db.tables.daos.ImporterMappingWorkflowDao; import org.jooq.Configuration; import org.jooq.DSLContext; -import org.jooq.Record; import javax.inject.Inject; -import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.UUID; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 2c3f7854f..ef6c5f884 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -440,7 +440,7 @@ private void processFile(UUID workflowId, List finalBrAPIImportList Optional optionalWorkflow = workflowFactory.getWorkflow(workflowId); workflow = optionalWorkflow.orElse(null); } - + ImportServiceContext context = ImportServiceContext.builder() .workflow(workflow) .brAPIImports(finalBrAPIImportList) From 87ffdb53d134b43b0955aacee063c351d29d5cfd Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 13 May 2024 17:08:39 -0400 Subject: [PATCH 047/203] Add bean to import mapping workflow model --- .../brapps/importer/model/workflow/ImportMappingWorkflow.java | 4 ++++ .../brapps/importer/model/workflow/ProcessedData.java | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java index 0a23e1aae..4ea2a888b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java @@ -36,15 +36,19 @@ @NoArgsConstructor @SuperBuilder() public class ImportMappingWorkflow extends ImporterMappingWorkflowEntity { + + public ImportMappingWorkflow(ImporterMappingWorkflowEntity importMappingWorkflowEntity) { this.setId(importMappingWorkflowEntity.getId()); this.setName(importMappingWorkflowEntity.getName()); + this.setBean(importMappingWorkflowEntity.getBean()); } public static ImportMappingWorkflow parseSQLRecord(Record record) { return ImportMappingWorkflow.builder() .id(record.getValue(IMPORTER_MAPPING_WORKFLOW.ID)) .name(record.getValue(IMPORTER_MAPPING_WORKFLOW.NAME)) + .bean(record.getValue(IMPORTER_MAPPING_WORKFLOW.BEAN)) .build(); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java index e73d6ad48..f9f8196c2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -23,7 +23,6 @@ import java.util.Map; @Data -@Builder @ToString @NoArgsConstructor public class ProcessedData { From 19e3f17a90eac5883644c3278df4ef4beff93c78 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 13 May 2024 18:24:18 -0400 Subject: [PATCH 048/203] validate dynamic columns --- .../model/v1/response/ValidationErrors.java | 2 +- .../middleware/process/TraitVerification.java | 97 ++++-- .../appendoverwrite/model/ExpUnitContext.java | 2 + .../experiment/create/model/PendingData.java | 2 + .../model/ExpImportProcessConstants.java | 4 +- .../service/ObservationVariableService.java | 288 ++++++++++++++++++ 6 files changed, 361 insertions(+), 34 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java diff --git a/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java b/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java index 3c7f1d706..923529505 100644 --- a/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java +++ b/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java @@ -17,6 +17,7 @@ package org.breedinginsight.api.model.v1.response; +import io.micronaut.http.HttpStatus; import lombok.*; import lombok.experimental.Accessors; @@ -46,7 +47,6 @@ public void addError(Integer rowNumber, ValidationError validationError){ newRow.addError(validationError); rowErrors.add(newRow); } - public void merge(ValidationErrors validationErrors){ for (RowValidationErrors rowValidationErrors: validationErrors.getRowErrors()){ for (ValidationError validationError: rowValidationErrors.getErrors()) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java index facbe014f..5801f47d6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java @@ -1,60 +1,93 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.services.FileMappingUtil; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationVariableService; +import org.breedinginsight.dao.db.tables.pojos.TraitEntity; import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import tech.tablesaw.api.Table; +import tech.tablesaw.columns.Column; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.api.model.v1.response.ValidationErrors; import javax.inject.Inject; import java.util.*; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_REGEX; + @Slf4j public class TraitVerification extends ExpUnitMiddleware { - ObservationUnitService observationUnitService; + ObservationVariableService observationVariableService; + FileMappingUtil fileMappingUtil; @Inject - public TraitVerification(ObservationUnitService observationUnitService) { - this.observationUnitService = observationUnitService; + public TraitVerification(ObservationVariableService observationVariableService, + FileMappingUtil fileMappingUtil) { + this.observationVariableService = observationVariableService; + this.fileMappingUtil = fileMappingUtil; } @Override public boolean process(ExpUnitMiddlewareContext context) { - Program program; - Set expUnitIds; - List missingIds; - List brapiUnits; - List> pendingUnits; - Map> pendingUnitById; - Map> pendingUnitByNameNoScope; - - log.debug("fetching required exp units from BrAPI service"); - program = context.getImportContext().getProgram(); - - // Collect deltabreed-generated exp unit ids listed in the import - expUnitIds = context.getExpUnitContext().getReferenceOUIds(); - try { - // For each id fetch the observation unit from the brapi data store - brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); + log.debug("verifying traits listed in import"); + + // Get all the dynamic columns of the import + ImportUpload upload = context.getImportContext().getUpload(); + Table data = context.getImportContext().getData(); + String[] dynamicColNames = upload.getDynamicColumnNames(); + List> dynamicCols = data.columns(dynamicColNames); + + // Collect the columns for observation variable data + List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); + Set varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toSet()); + + // Collect the columns for observation timestamps + List> timestampCols = dynamicCols.stream().filter(col -> col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); + Set tsNames = timestampCols.stream().map(Column::name).collect(Collectors.toSet()); + + // Construct validation errors for any timestamp columns that don't have a matching variable column + List importRows = context.getImportContext().getImportRows(); + ValidationErrors validationErrors = context.getPendingData().getValidationErrors(); + List tsValErrs = observationVariableService.validateMatchedTimestamps(varNames, timestampCols).orElse(new ArrayList<>()); + for (int rowNum = 0; rowNum < importRows.size(); i++) { + tsValErrs.forEach(validationError -> validationErrors.addError(rowNum, validationError)); + } + + // Stop processing the import if there are unmatched timestamp columns + if (tsValErrs.size() > 0) { + this.compensate(context, new MiddlewareError(() -> { + // any handling... + })); + } - // Construct pending import objects from the units - pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); + //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval + Map> colByPheno = timestampCols.stream().collect(Collectors.toMap(col -> col.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), col -> col)); - // Construct a hashmap to look up the pending unit by ID - pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); + // Add the map to the context for use in processing import + context.getPendingData().setTimeStampColByPheno(colByPheno); - // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed - pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); + try { + // Fetch the traits named in the observation variable columns + Program program = context.getImportContext().getProgram(); + List traits = observationVariableService.fetchTraitsByName(varNames, program); - // add maps to the context for use in processing import - context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); - context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); - } catch (ApiException e) { + // Sort the traits to match the order of the headers in the import file + List sortedTraits = fileMappingUtil.sortByField(List.copyOf(varNames), new ArrayList<>(traits), TraitEntity::getObservationVariableName); + } catch (DoesNotExistException e) { this.compensate(context, new MiddlewareError(() -> { throw new RuntimeException(e); })); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java index 50ca1f4d2..6988cbf89 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java @@ -7,8 +7,10 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.model.ProgramLocation; +import tech.tablesaw.columns.Column; import java.util.HashMap; import java.util.HashSet; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java index 340a39fba..58f97f727 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -9,6 +9,7 @@ import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.model.ProgramLocation; +import tech.tablesaw.columns.Column; import java.util.HashMap; import java.util.Map; @@ -26,5 +27,6 @@ public class PendingData { private Map> locationByName; private Map> obsVarDatasetByName; private Map> existingGermplasmByGID; + private Map> timeStampColByPheno; private ValidationErrors validationErrors; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java index 4da4414a1..a81a5067e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java @@ -5,10 +5,12 @@ public class ExpImportProcessConstants { public static final CharSequence COMMA_DELIMITER = ","; + public static final String TIMESTAMP_PREFIX = "TS:"; + public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; public enum ErrMessage { - MISSING_OBS_UNIT_ID_ERROR("Experimental entities are missing ObsUnitIDs"), PREEXISTING_EXPERIMENT_TITLE("Experiment Title already exists"); + private String value; ErrMessage(String value) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java new file mode 100644 index 000000000..bd4ced017 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java @@ -0,0 +1,288 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.server.exceptions.InternalServerException; +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.Utilities; +import tech.tablesaw.columns.Column; +import org.breedinginsight.dao.db.tables.pojos.TraitEntity; + +import javax.inject.Inject; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.stream.Collectors; + +public class ObservationVariableService { + private final OntologyService ontologyService; + + @Inject + public ObservationVariableService(OntologyService ontologyService) { + this.ontologyService = ontologyService; + } + + /** + * Fetches traits by name for the given set of variable names and program. + * + * This method fetches all stored traits for the specified program and filters them based on the set of variable names provided. + * It ensures that all requested observation variables are present and returns a list of matching traits. + * If any observation variables are missing, it throws an IllegalStateException with the missing variable names. + * + * @param varNames a set of variable names to fetch traits for + * @param program the program for which traits are fetched + * @return a list of traits filtered by the provided variable names + * @throws DoesNotExistException if the program or traits do not exist + * @throws IllegalStateException if any requested observation variables are missing + */ + public List fetchTraitsByName(Set varNames, Program program) throws DoesNotExistException, IllegalStateException { + List traits = null; + + // Fetch all stored traits for the program + List programTraits = ontologyService.getTraitsByProgramId(program.getId(), true); + + // Only keep traits that are in the set of names + List upperCaseVarNames = varNames.stream().map(String::toUpperCase).collect(Collectors.toList()); + traits = programTraits.stream().filter(e -> upperCaseVarNames.contains(e.getObservationVariableName().toUpperCase())).collect(Collectors.toList()); + + // If any requested observation variables are missing, throw an IllegalStateException + if (varNames.size() != traits.size()) { + Set missingVarNames = new HashSet<>(varNames); + missingVarNames.removeAll(traits.stream().map(TraitEntity::getObservationVariableName).collect(Collectors.toSet())); + throw new IllegalStateException("Observation variables not found for name(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingVarNames)); + } + + return traits; + } + + public Optional> validateMatchedTimestamps(Set observationVariableNames, + List> timestampCols) { + Optional> ve = Optional.empty(); + // Check that each ts column corresponds to a phenotype column + List valErrs = timestampCols.stream() + .filter(col -> !(observationVariableNames.contains(col.name().replaceFirst(ExpImportProcessConstants.TIMESTAMP_REGEX, StringUtils.EMPTY)))) + .map(col -> new ValidationError(col.name().toString(), String.format("Timestamp column %s lacks corresponding phenotype column", col.name().toString()), HttpStatus.UNPROCESSABLE_ENTITY)) + .collect(Collectors.toList()); + if (valErrs.size() > 0) { + ve = Optional.of(valErrs); + } + + return ve; + } + // TODO: used with expUnit workflow + public void validateObservations(PendingData pendingData, + int rowNum, + ImportContext importContext, + ExpUnitContext expUnitContext, + List> phenotypeCols, + CaseInsensitiveMap colVarMap) { + for (Column phenoCol : phenotypeCols) { + String importHash; + String importObsValue = phenoCol.getString(rowNum); + + importHash = getImportObservationHash( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(phenoCol), + pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() + ); + + validateObservation(importHash); + } + } + + // TODO: used with create workflow + public void validateObservations(PendingData pendingData, + int rowNum, + ImportContext importContext, + List> phenotypeCols, + CaseInsensitiveMap colVarMap) { + + + for (Column phenoCol : phenotypeCols) { + String importHash; + String importObsValue = phenoCol.getString(rowNum); + + importHash = getImportObservationHash(importRow, phenoCol.name()); + + validateObservation(importHash); + } + + } + + // TODO: used by both workflows + private void validateObservation(String importHash) { + + + String importObsValue = phenoCol.getString(rowNum); + + + // error if import observation data already exists and user has not selected to overwrite + if (commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && + this.existingObsByObsHash.containsKey(importHash) && + StringUtils.isNotBlank(phenoCol.getString(rowNum)) && + !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { + addRowError( + phenoCol.name(), + String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), + validationErrors, rowNum + ); + + // preview case where observation has already been committed and the import row ObsVar data differs from what + // had been saved prior to import + } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { + + // add a change log entry when updating the value of an observation + if (commit) { + BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; + String prior = ""; + if (isValueMatched(importHash, importObsValue)) { + prior.concat(existingObsByObsHash.get(importHash).getValue()); + } + if (timeStampColByPheno.containsKey(phenoCol.name()) && isTimestampMatched(importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { + prior = prior.isEmpty() ? prior : prior.concat(" "); + prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); + } + ChangeLogEntry change = new ChangeLogEntry(prior, + reason, + user.getId(), + timestamp + ); + + // create the changelog field in additional info if it does not already exist + if (pendingObservation.getAdditionalInfo().isJsonNull()) { + pendingObservation.setAdditionalInfo(new JsonObject()); + pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + + if (pendingObservation.getAdditionalInfo() != null && !pendingObservation.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { + pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + + // add a new entry to the changelog + pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); + } + + // preview case where observation has already been committed and import ObsVar data is the + // same as has been committed prior to import + } else if (isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { + BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); + existingObs.setObservationVariableName(phenoCol.name()); + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + observationByHash.get(importHash).setBrAPIObject(existingObs); + + // preview case where observation has already been committed and import ObsVar data is empty prior to import + } else if (!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + } else { + validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); + + //Timestamp validation + if (timeStampColByPheno.containsKey(phenoCol.name())) { + Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); + validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); + } + } + + } + + // TODO: used by both workflows + private void updateObservationDependencyValues(Program program) { + String programKey = program.getKey(); + + // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds + this.observationUnitByNameNoScope.values().stream() + .map(PendingImportObject::getBrAPIObject) + .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); + + // Update ObservationVariable DbIds + List traits = getTraitList(program); + CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); + for ( Trait trait: traits) { + traitMap.put(trait.getObservationVariableName(),trait); + } + for (PendingImportObject observation : this.observationByHash.values()) { + String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); + if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { + String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); + observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); + } + } + } + + // TODO: used by both workflows + public List commitNewPendingObservationsToBrAPIStore(ImportContext context, PendingData pendingData) { + // filter out observations with no 'value' so they will not be saved + List newObservations = ProcessorData.getNewObjects(this.observationByHash) + .stream() + .filter(obs -> !obs.getValue().isBlank()) + .collect(Collectors.toList()); + + updateObservationDependencyValues(program); + return brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); + + } + + // TODO: used by both workflows + public List commitUpdatedPendingObservationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedObservations = new ArrayList<>(); + Map mutatedObservationByDbId = ProcessorData + .getMutationsByObjectId(observationByHash, BrAPIObservation::getObservationDbId); + + for (Map.Entry entry : mutatedObservationByDbId.entrySet()) { + String id = entry.getKey(); + BrAPIObservation observation = entry.getValue(); + try { + if (observation == null) { + throw new Exception("Null observation"); + } + BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); + updatedObservations.add(updatedObs); + + if (updatedObs == null) { + throw new Exception("Null updated observation"); + } + + if (!Objects.equals(observation.getValue(), updatedObs.getValue()) + || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { + String message; + if (!Objects.equals(observation.getValue(), updatedObs.getValue())) { + message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); + } else { + message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); + } + throw new Exception(message); + } + + } catch (ApiException e) { + log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating observation: ", e); + throw new InternalServerException(e.getMessage(), e); + } + } + + return updatedObservations; + } +} From 964c2f3e9b707dfd330f24dceb0349da5f010077 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 14 May 2024 09:57:25 -0400 Subject: [PATCH 049/203] hash observations and add map to context --- .../middleware/process/TraitVerification.java | 45 +++++++++++++-- .../experiment/create/model/PendingData.java | 2 + .../service/ObservationService.java | 56 +++++++++++++++++-- .../experiment/service/StudyService.java | 1 + 4 files changed, 94 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java index 5801f47d6..2c0c3a913 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java @@ -1,27 +1,31 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; -import io.micronaut.http.HttpStatus; -import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.FileMappingUtil; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationVariableService; import org.breedinginsight.dao.db.tables.pojos.TraitEntity; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; -import org.breedinginsight.services.OntologyService; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.Utilities; import tech.tablesaw.api.Table; import tech.tablesaw.columns.Column; -import org.breedinginsight.api.model.v1.response.ValidationError; -import org.breedinginsight.api.model.v1.response.ValidationErrors; import javax.inject.Inject; import java.util.*; @@ -32,12 +36,18 @@ @Slf4j public class TraitVerification extends ExpUnitMiddleware { ObservationVariableService observationVariableService; + ObservationService observationService; + BrAPIObservationDAO brAPIObservationDAO; FileMappingUtil fileMappingUtil; @Inject public TraitVerification(ObservationVariableService observationVariableService, + BrAPIObservationDAO brAPIObservationDAO, + ObservationService observationService, FileMappingUtil fileMappingUtil) { this.observationVariableService = observationVariableService; + this.brAPIObservationDAO = brAPIObservationDAO; + this.observationService = observationService; this.fileMappingUtil = fileMappingUtil; } @@ -87,7 +97,30 @@ public boolean process(ExpUnitMiddlewareContext context) { // Sort the traits to match the order of the headers in the import file List sortedTraits = fileMappingUtil.sortByField(List.copyOf(varNames), new ArrayList<>(traits), TraitEntity::getObservationVariableName); - } catch (DoesNotExistException e) { + + // Read any observation data stored for these traits + Set ouDbIds = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(u -> u.getBrAPIObject().getObservationUnitDbId()).collect(Collectors.toSet()); + Set varDbIds = sortedTraits.stream().map(t->t.getObservationVariableDbId()).collect(Collectors.toSet()); + List observations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, varDbIds, program); + + // Construct helper lookup tables to use for hashing observations + Map unitNameByDbId = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, BrAPIObservationUnit::getObservationUnitName)); + Map variableNameByDbId = sortedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableDbId, Trait::getObservationVariableName)); + Map studyNameByDbId = context.getPendingData().getStudyByNameNoScope().values().stream() + .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) + .map(PendingImportObject::getBrAPIObject) + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); + + // Hash observations using a signature of unit, variable, and study names + Map observationByObsHash = observations.stream().collect(Collectors.toMap(o->{ + return observationService.getObservationHash(unitNameByDbId.get(o.getObservationUnitDbId()), + variableNameByDbId.get(o.getObservationVariableDbId()), + studyNameByDbId.get(o.getStudyDbId())); + }, o->o)); + + // Add the observation map to the context for use in processing import + context.getPendingData().setExistingObsByObsHash(observationByObsHash); + } catch (DoesNotExistException | ApiException e) { this.compensate(context, new MiddlewareError(() -> { throw new RuntimeException(e); })); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java index 58f97f727..7799e9574 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -5,6 +5,7 @@ import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -28,5 +29,6 @@ public class PendingData { private Map> obsVarDatasetByName; private Map> existingGermplasmByGID; private Map> timeStampColByPheno; + private Map existingObsByObsHash; private ValidationErrors validationErrors; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index f3ac20000..e421fa54d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -3,9 +3,11 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; import io.micronaut.http.server.exceptions.InternalServerException; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; @@ -22,13 +24,59 @@ import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.stream.Collectors; public class ObservationService { + public String getObservationHash(String observationUnitName, String variableName, String studyName) { + String concat = DigestUtils.sha256Hex(observationUnitName) + + DigestUtils.sha256Hex(variableName) + + DigestUtils.sha256Hex(StringUtils.defaultString(studyName)); + return DigestUtils.sha256Hex(concat); + } + public Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { + Set ouDbIds = new HashSet<>(); + Set variableDbIds = new HashSet<>(); + Map variableNameByDbId = new HashMap<>(); + Map ouNameByDbId = new HashMap<>(); + Map studyNameByDbId = studyByNameNoScope.values() + .stream() + .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) + .map(PendingImportObject::getBrAPIObject) + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); + + studyNameByDbId.keySet().forEach(studyDbId -> { + try { + brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { + if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { + ouDbIds.add(ou.getObservationUnitDbId()); + } + ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + }); + } catch (ApiException e) { + throw new RuntimeException(e); + } + }); + + for (Trait referencedTrait : referencedTraits) { + variableDbIds.add(referencedTrait.getObservationVariableDbId()); + variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); + } + + List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); + + return existingObservations.stream() + .map(obs -> { + String studyName = studyNameByDbId.get(obs.getStudyDbId()); + String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); + String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); + + String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); + + return Map.entry(key, obs); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } // TODO: used with expUnit workflow public void validateObservations(PendingData pendingData, int rowNum, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index baf646689..53015159b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -52,6 +52,7 @@ public StudyService(BrAPISeasonDAO brAPISeasonDAO, this.brAPIStudyDAO = brAPIStudyDAO; } + // TODO: used by both workflows public PendingImportObject processAndCacheStudy( BrAPIStudy existingStudy, From 186ee1d893d47ccb2d3290e6374df106d52f4cdd Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 14 May 2024 10:00:48 -0400 Subject: [PATCH 050/203] add logging statement --- .../appendoverwrite/middleware/process/TraitVerification.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java index 2c0c3a913..59936a333 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java @@ -99,6 +99,7 @@ public boolean process(ExpUnitMiddlewareContext context) { List sortedTraits = fileMappingUtil.sortByField(List.copyOf(varNames), new ArrayList<>(traits), TraitEntity::getObservationVariableName); // Read any observation data stored for these traits + log.debug("fetching observation data stored for traits"); Set ouDbIds = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(u -> u.getBrAPIObject().getObservationUnitDbId()).collect(Collectors.toSet()); Set varDbIds = sortedTraits.stream().map(t->t.getObservationVariableDbId()).collect(Collectors.toSet()); List observations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, varDbIds, program); From dc2ea96c6879ce858f8f817f4bcc5f932674df5b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 14 May 2024 10:08:06 -0400 Subject: [PATCH 051/203] fix for loop --- .../appendoverwrite/middleware/process/TraitVerification.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java index 59936a333..a2a04194d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java @@ -73,7 +73,8 @@ public boolean process(ExpUnitMiddlewareContext context) { List importRows = context.getImportContext().getImportRows(); ValidationErrors validationErrors = context.getPendingData().getValidationErrors(); List tsValErrs = observationVariableService.validateMatchedTimestamps(varNames, timestampCols).orElse(new ArrayList<>()); - for (int rowNum = 0; rowNum < importRows.size(); i++) { + for (int i = 0; i < importRows.size(); i++) { + int rowNum = i; tsValErrs.forEach(validationError -> validationErrors.addError(rowNum, validationError)); } From 1c1df39b03dd513adff46bc8bd7e8d9dde19bff7 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 14 May 2024 10:29:04 -0400 Subject: [PATCH 052/203] Added position column to workflows for explicit ordering --- .../daos/ImportMappingWorkflowDAO.java | 4 +++- .../model/workflow/ImportMappingWorkflow.java | 2 ++ .../services/workflow/WorkflowFactory.java | 4 +++- .../V1.22.0__add_experiment_workflows.sql | 19 ++++++++++++++----- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java index 216506b5c..1b60bda45 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java @@ -40,7 +40,8 @@ public ImportMappingWorkflowDAO(Configuration config, DSLContext dsl) { } /** - * Retrieves a list of ImportMappingWorkflow objects associated with the given mappingId. + * Retrieves a list of ImportMappingWorkflow objects associated with the given mappingId. They are ordered by + * position for proper ordering on the front end. * * @param mappingId The UUID of the mapping to retrieve the workflows for. * @return A list of ImportMappingWorkflow objects. @@ -49,6 +50,7 @@ public List getWorkflowsByImportMappingId(UUID mappingId) return dsl.select() .from(IMPORTER_MAPPING_WORKFLOW) .where(IMPORTER_MAPPING_WORKFLOW.MAPPING_ID.eq(mappingId)) + .orderBy(IMPORTER_MAPPING_WORKFLOW.POSITION.asc()) .fetch(ImportMappingWorkflow::parseSQLRecord); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java index 4ea2a888b..8dc5800bc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java @@ -42,6 +42,7 @@ public ImportMappingWorkflow(ImporterMappingWorkflowEntity importMappingWorkflow this.setId(importMappingWorkflowEntity.getId()); this.setName(importMappingWorkflowEntity.getName()); this.setBean(importMappingWorkflowEntity.getBean()); + this.setPosition(importMappingWorkflowEntity.getPosition()); } public static ImportMappingWorkflow parseSQLRecord(Record record) { @@ -49,6 +50,7 @@ public static ImportMappingWorkflow parseSQLRecord(Record record) { .id(record.getValue(IMPORTER_MAPPING_WORKFLOW.ID)) .name(record.getValue(IMPORTER_MAPPING_WORKFLOW.NAME)) .bean(record.getValue(IMPORTER_MAPPING_WORKFLOW.BEAN)) + .position(record.getValue(IMPORTER_MAPPING_WORKFLOW.POSITION)) .build(); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java index 60a2fd884..d265b66a0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java @@ -48,7 +48,7 @@ public WorkflowFactory(ImportMappingWorkflowDAO importMappingWorkflowDAO, * Produces the appropriate workflow instance based on the import context * * @param context the import context - * @return an Optional containing the workflow if found, otherwise an empty Optional + * @return an Optional containing the workflow if id is not null, otherwise an empty Optional * * @throws IllegalStateException * @throws NoSuchBeanException @@ -58,6 +58,8 @@ public Optional getWorkflow(UUID workflowId) { if (workflowId != null) { // construct workflow from db record Optional workflowOptional = importMappingWorkflowDAO.getWorkflowById(workflowId); + + // TODO: look at this optional handling if (workflowOptional.isPresent()) { ImportMappingWorkflow importMappingworkflow = workflowOptional.orElseThrow(() -> { String msg = "Must have record in db for workflowId"; diff --git a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql index d5c4b9556..2c9d4d547 100644 --- a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql +++ b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql @@ -15,12 +15,21 @@ * limitations under the License. */ +/** + Table maps workflows to import mappings and provides required configuration options + + mapping_id - link to importer_mapping that this provides a workflow for + name - name that will be displayed on front end + bean - must match @Named("") annotation on Workflow class + position - for ordering records explicitly, wanted for front end default option and order + */ CREATE TABLE importer_mapping_workflow ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, mapping_id UUID NOT NULL, name TEXT NOT NULL, - bean TEXT NOT NULL + bean TEXT NOT NULL, + position INTEGER NOT NULL ); ALTER TABLE importer_mapping_workflow @@ -33,10 +42,10 @@ DECLARE BEGIN exp_mapping_id := (SELECT id FROM importer_mapping WHERE name = 'ExperimentsTemplateMap'); -INSERT INTO public.importer_mapping_workflow (mapping_id, name, bean) +INSERT INTO public.importer_mapping_workflow (mapping_id, name, bean, position) VALUES - (exp_mapping_id, 'Create new experiment', 'CreateNewExperimentWorkflow'), - (exp_mapping_id, 'Append experimental dataset', 'AppendOverwritePhenotypesWorkflow'), - (exp_mapping_id, 'Create new experimental environment', 'CreateNewEnvironmentWorkflow'); + (exp_mapping_id, 'Create new experiment', 'CreateNewExperimentWorkflow', 0), + (exp_mapping_id, 'Append experimental dataset', 'AppendOverwritePhenotypesWorkflow', 1), + (exp_mapping_id, 'Create new experimental environment', 'CreateNewEnvironmentWorkflow', 2); END $$; \ No newline at end of file From 36944a510a2dade39bd1dff565621a1b2f6f6e41 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 14 May 2024 10:45:35 -0400 Subject: [PATCH 053/203] Added additional workflow skeletons --- .../AppendOverwritePhenotypesWorkflow.java | 50 +++++++++++++++++++ .../CreateNewEnvironmentWorkflow.java | 50 +++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java new file mode 100644 index 000000000..21df19be6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -0,0 +1,50 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; + +import javax.inject.Named; + +/** + * This class represents a workflow for appending and overwriting phenotypes. The bean name must match the appropriate + * bean column value in the import_mapping_workflow db table + */ + +@Prototype +@Named("AppendOverwritePhenotypesWorkflow") +public class AppendOverwritePhenotypesWorkflow implements Workflow { + @Override + public ProcessedData process(ImportContext context) { + // TODO + return null; + } + + /** + * Retrieves the name of the workflow. This is used for logging display purposes. + * + * @return the name of the workflow + */ + @Override + public String getName() { + return "AppendOverwritePhenotypesWorkflow"; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java new file mode 100644 index 000000000..5776ab59b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java @@ -0,0 +1,50 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; + +import javax.inject.Named; + +/** + * This class represents a workflow for adding new environments to an existing experiment. The bean name must match + * the appropriate bean column value in the import_mapping_workflow db table + */ + +@Prototype +@Named("CreateNewEnvironmentWorkflow") +public class CreateNewEnvironmentWorkflow implements Workflow { + @Override + public ProcessedData process(ImportContext context) { + // TODO + return null; + } + + /** + * Retrieves the name of the workflow. This is used for logging display purposes. + * + * @return the name of the workflow + */ + @Override + public String getName() { + return "CreateNewEnvironmentWorkflow"; + } +} From 7a9af10ed0356559c6a27de474c86697f0e38f58 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 14 May 2024 14:01:41 -0400 Subject: [PATCH 054/203] Clean up factory code --- .../services/workflow/WorkflowFactory.java | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java index d265b66a0..17d6fe64e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java @@ -59,26 +59,23 @@ public Optional getWorkflow(UUID workflowId) { // construct workflow from db record Optional workflowOptional = importMappingWorkflowDAO.getWorkflowById(workflowId); - // TODO: look at this optional handling - if (workflowOptional.isPresent()) { - ImportMappingWorkflow importMappingworkflow = workflowOptional.orElseThrow(() -> { - String msg = "Must have record in db for workflowId"; - log.error(msg); - return new IllegalStateException(msg); - }); + ImportMappingWorkflow importMappingworkflow = workflowOptional.orElseThrow(() -> { + String msg = "Must have record in db for workflowId"; + log.error(msg); + return new IllegalStateException(msg); + }); - // newer versions of micronaut have fancier ways to do this using annotations with provider but as - // far as I can tell it's not available in 2.5 - Workflow workflow; - try { - workflow = applicationContext.getBean(Workflow.class, Qualifiers.byName(importMappingworkflow.getBean())); - } catch (NoSuchBeanException e) { - log.error("Could not find workflow class implementation for bean: " + importMappingworkflow.getBean()); - throw e; - } - - return Optional.of(workflow); + // newer versions of micronaut have fancier ways to do this using annotations with provider but as + // far as I can tell it's not available in 2.5 + Workflow workflow; + try { + workflow = applicationContext.getBean(Workflow.class, Qualifiers.byName(importMappingworkflow.getBean())); + } catch (NoSuchBeanException e) { + log.error("Could not find workflow class implementation for bean: " + importMappingworkflow.getBean()); + throw e; } + + return Optional.of(workflow); } return Optional.empty(); From 027855f3bf43618ce0f54b811387247d0cb66057 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 15 May 2024 15:19:08 -0400 Subject: [PATCH 055/203] construct pending observations --- .../experiment/ExperimentUtilities.java | 15 +- .../process/ImportPreviewStatistics.java | 7 +- .../process/NewPendingBrAPIObjects.java | 65 ----- .../middleware/process/TraitVerification.java | 133 --------- .../process/brapi/NewPendingBrAPIObjects.java | 49 ++++ .../process/brapi/PendingObservation.java | 254 ++++++++++++++++++ .../read/brapi/RequiredDatasets.java | 25 +- .../experiment/create/model/PendingData.java | 1 + .../model/ExpImportProcessConstants.java | 5 + .../experiment/service/DatasetService.java | 8 +- .../service/ObservationService.java | 20 ++ .../experiment/service/StudyService.java | 5 +- 12 files changed, 360 insertions(+), 227 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/NewPendingBrAPIObjects.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index d22f9ae2f..e321c2f59 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -12,10 +12,9 @@ import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.*; import java.util.stream.Collectors; public class ExperimentUtilities { @@ -52,6 +51,14 @@ public V getSingleEntryValue(Map map, String message) throws Unproc return map.values().iterator().next(); } + public static Optional getSingleEntryValue(Map map) { + Optional value = Optional.empty(); + if (map.size() == 1) { + value = Optional.ofNullable(map.values().iterator().next()); + } + return value; + } + /* * this will add the given year to the additionalInfo field of the BrAPIStudy (if it does not already exist) * */ diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java index a2a446f4c..0ead6a47e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java @@ -2,6 +2,8 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi.NewPendingBrAPIObjects; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi.PendingObservation; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import javax.inject.Inject; @@ -10,19 +12,16 @@ @Slf4j public class ImportPreviewStatistics extends ExpUnitMiddleware { ExpUnitMiddleware middleware; - private Provider traitVerificationProvider; private Provider newPendingBrAPIObjectsProvider; private Provider dataValidationProvider; private Provider fieldValidationProvider; @Inject - public ImportPreviewStatistics(Provider traitVerificationProvider, - Provider newPendingBrAPIObjectsProvider, + public ImportPreviewStatistics(Provider newPendingBrAPIObjectsProvider, Provider dataValidationProvider, Provider fieldValidationProvider) { this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( - traitVerificationProvider.get(), // Verify observation variable columns in import belong to program newPendingBrAPIObjectsProvider.get(), // Construct Pending import objects for new BrAPI data dataValidationProvider.get(), // Validate data fieldValidationProvider.get()); // Validate fields diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/NewPendingBrAPIObjects.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/NewPendingBrAPIObjects.java deleted file mode 100644 index fc6b63f98..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/NewPendingBrAPIObjects.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; - -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; -import org.breedinginsight.model.Program; - -import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; - -@Slf4j -public class NewPendingBrAPIObjects extends ExpUnitMiddleware { - ObservationUnitService observationUnitService; - - @Inject - public NewPendingBrAPIObjects(ObservationUnitService observationUnitService) { - this.observationUnitService = observationUnitService; - } - - @Override - public boolean process(ExpUnitMiddlewareContext context) { - Program program; - Set expUnitIds; - List missingIds; - List brapiUnits; - List> pendingUnits; - Map> pendingUnitById; - Map> pendingUnitByNameNoScope; - - log.debug("fetching required exp units from BrAPI service"); - program = context.getImportContext().getProgram(); - - // Collect deltabreed-generated exp unit ids listed in the import - expUnitIds = context.getExpUnitContext().getReferenceOUIds(); - try { - // For each id fetch the observation unit from the brapi data store - brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); - - // Construct pending import objects from the units - pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); - - // Construct a hashmap to look up the pending unit by ID - pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); - - // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed - pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); - - // add maps to the context for use in processing import - context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); - context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); - } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); - } - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java deleted file mode 100644 index a2a04194d..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java +++ /dev/null @@ -1,133 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.pheno.BrAPIObservation; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.api.model.v1.response.ValidationError; -import org.breedinginsight.api.model.v1.response.ValidationErrors; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.FileMappingUtil; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationVariableService; -import org.breedinginsight.dao.db.tables.pojos.TraitEntity; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.Trait; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.utilities.Utilities; -import tech.tablesaw.api.Table; -import tech.tablesaw.columns.Column; - -import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; - -import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_REGEX; - -@Slf4j -public class TraitVerification extends ExpUnitMiddleware { - ObservationVariableService observationVariableService; - ObservationService observationService; - BrAPIObservationDAO brAPIObservationDAO; - FileMappingUtil fileMappingUtil; - - @Inject - public TraitVerification(ObservationVariableService observationVariableService, - BrAPIObservationDAO brAPIObservationDAO, - ObservationService observationService, - FileMappingUtil fileMappingUtil) { - this.observationVariableService = observationVariableService; - this.brAPIObservationDAO = brAPIObservationDAO; - this.observationService = observationService; - this.fileMappingUtil = fileMappingUtil; - } - - @Override - public boolean process(ExpUnitMiddlewareContext context) { - log.debug("verifying traits listed in import"); - - // Get all the dynamic columns of the import - ImportUpload upload = context.getImportContext().getUpload(); - Table data = context.getImportContext().getData(); - String[] dynamicColNames = upload.getDynamicColumnNames(); - List> dynamicCols = data.columns(dynamicColNames); - - // Collect the columns for observation variable data - List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); - Set varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toSet()); - - // Collect the columns for observation timestamps - List> timestampCols = dynamicCols.stream().filter(col -> col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); - Set tsNames = timestampCols.stream().map(Column::name).collect(Collectors.toSet()); - - // Construct validation errors for any timestamp columns that don't have a matching variable column - List importRows = context.getImportContext().getImportRows(); - ValidationErrors validationErrors = context.getPendingData().getValidationErrors(); - List tsValErrs = observationVariableService.validateMatchedTimestamps(varNames, timestampCols).orElse(new ArrayList<>()); - for (int i = 0; i < importRows.size(); i++) { - int rowNum = i; - tsValErrs.forEach(validationError -> validationErrors.addError(rowNum, validationError)); - } - - // Stop processing the import if there are unmatched timestamp columns - if (tsValErrs.size() > 0) { - this.compensate(context, new MiddlewareError(() -> { - // any handling... - })); - } - - //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval - Map> colByPheno = timestampCols.stream().collect(Collectors.toMap(col -> col.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), col -> col)); - - // Add the map to the context for use in processing import - context.getPendingData().setTimeStampColByPheno(colByPheno); - - try { - // Fetch the traits named in the observation variable columns - Program program = context.getImportContext().getProgram(); - List traits = observationVariableService.fetchTraitsByName(varNames, program); - - // Sort the traits to match the order of the headers in the import file - List sortedTraits = fileMappingUtil.sortByField(List.copyOf(varNames), new ArrayList<>(traits), TraitEntity::getObservationVariableName); - - // Read any observation data stored for these traits - log.debug("fetching observation data stored for traits"); - Set ouDbIds = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(u -> u.getBrAPIObject().getObservationUnitDbId()).collect(Collectors.toSet()); - Set varDbIds = sortedTraits.stream().map(t->t.getObservationVariableDbId()).collect(Collectors.toSet()); - List observations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, varDbIds, program); - - // Construct helper lookup tables to use for hashing observations - Map unitNameByDbId = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, BrAPIObservationUnit::getObservationUnitName)); - Map variableNameByDbId = sortedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableDbId, Trait::getObservationVariableName)); - Map studyNameByDbId = context.getPendingData().getStudyByNameNoScope().values().stream() - .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) - .map(PendingImportObject::getBrAPIObject) - .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); - - // Hash observations using a signature of unit, variable, and study names - Map observationByObsHash = observations.stream().collect(Collectors.toMap(o->{ - return observationService.getObservationHash(unitNameByDbId.get(o.getObservationUnitDbId()), - variableNameByDbId.get(o.getObservationVariableDbId()), - studyNameByDbId.get(o.getStudyDbId())); - }, o->o)); - - // Add the observation map to the context for use in processing import - context.getPendingData().setExistingObsByObsHash(observationByObsHash); - } catch (DoesNotExistException | ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); - } - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java new file mode 100644 index 000000000..8f7e6362a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java @@ -0,0 +1,49 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi; + +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.jooq.DSLContext; + +import javax.inject.Inject; +import javax.inject.Provider; +import java.math.BigInteger; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.MULTIPLE_EXP_TITLES; + +@Slf4j +public class NewPendingBrAPIObjects extends ExpUnitMiddleware { + ExpUnitMiddleware middleware; + Provider pendingObservationProvider; + @Inject + public NewPendingBrAPIObjects(Provider pendingObservationProvider) { + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(pendingObservationProvider.get()); // Construct new pending observation + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + log.debug("constructing new pending BrAPI objects"); + + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java new file mode 100644 index 000000000..162ee0842 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java @@ -0,0 +1,254 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi; + +import com.google.gson.Gson; +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.FileMappingUtil; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationVariableService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.dao.db.tables.pojos.TraitEntity; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; +import tech.tablesaw.api.Table; +import tech.tablesaw.columns.Column; + +import javax.inject.Inject; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.*; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.MULTIPLE_EXP_TITLES; + +@Slf4j +public class PendingObservation extends ExpUnitMiddleware { + StudyService studyService; + ObservationVariableService observationVariableService; + ObservationService observationService; + BrAPIObservationDAO brAPIObservationDAO; + FileMappingUtil fileMappingUtil; + Gson gson; + + @Inject + public PendingObservation(StudyService studyService, + ObservationVariableService observationVariableService, + BrAPIObservationDAO brAPIObservationDAO, + ObservationService observationService, + FileMappingUtil fileMappingUtil, + Gson gson) { + this.studyService = studyService; + this.observationVariableService = observationVariableService; + this.brAPIObservationDAO = brAPIObservationDAO; + this.observationService = observationService; + this.fileMappingUtil = fileMappingUtil; + this.gson = gson; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + log.debug("verifying traits listed in import"); + + // Get all the dynamic columns of the import + ImportUpload upload = context.getImportContext().getUpload(); + Table data = context.getImportContext().getData(); + String[] dynamicColNames = upload.getDynamicColumnNames(); + List> dynamicCols = data.columns(dynamicColNames); + + // Collect the columns for observation variable data + List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); + Set varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toSet()); + + // Collect the columns for observation timestamps + List> timestampCols = dynamicCols.stream().filter(col -> col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); + Set tsNames = timestampCols.stream().map(Column::name).collect(Collectors.toSet()); + + // Construct validation errors for any timestamp columns that don't have a matching variable column + List importRows = context.getImportContext().getImportRows(); + ValidationErrors validationErrors = context.getPendingData().getValidationErrors(); + List tsValErrs = observationVariableService.validateMatchedTimestamps(varNames, timestampCols).orElse(new ArrayList<>()); + for (int i = 0; i < importRows.size(); i++) { + int rowNum = i; + tsValErrs.forEach(validationError -> validationErrors.addError(rowNum, validationError)); + } + + // Stop processing the import if there are unmatched timestamp columns + if (tsValErrs.size() > 0) { + this.compensate(context, new MiddlewareError(() -> { + // any handling... + })); + } + + //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval + Map> tsColByPheno = timestampCols.stream().collect(Collectors.toMap(col -> col.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), col -> col)); + + // Add the map to the context for use in processing import + context.getPendingData().setTimeStampColByPheno(tsColByPheno); + + try { + // Fetch the traits named in the observation variable columns + Program program = context.getImportContext().getProgram(); + List traits = observationVariableService.fetchTraitsByName(varNames, program); + + // Sort the traits to match the order of the headers in the import file + List sortedTraits = fileMappingUtil.sortByField(List.copyOf(varNames), new ArrayList<>(traits), TraitEntity::getObservationVariableName); + + // Get the pending observation dataset + PendingImportObject pendingTrial = ExperimentUtilities.getSingleEntryValue(context.getPendingData().getTrialByNameNoScope()).orElseThrow(()->new UnprocessableEntityException(MULTIPLE_EXP_TITLES.getValue())); + String datasetName = String.format("Observation Dataset [%s-%s]", program.getKey(), pendingTrial.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString()); + PendingImportObject pendingDataset = context.getPendingData().getObsVarDatasetByName().get(datasetName); + + // Add new phenotypes to the pending observation dataset list (NOTE: "obsVarName [programKey]" is used instead of obsVarDbId) + // TODO: Change to using actual dbIds as per the BrAPI spec, instead of namespaced obsVar names (necessary for Breedbase) + Set datasetObsVarDbIds = pendingDataset.getBrAPIObject().getData().stream().collect(Collectors.toSet()); + Set phenoDbIds = sortedTraits.stream().map(t->Utilities.appendProgramKey(t.getObservationVariableName(), program.getKey())).collect(Collectors.toSet()); + phenoDbIds.removeAll(datasetObsVarDbIds); + pendingDataset.getBrAPIObject().getData().addAll(phenoDbIds); + + // Update pending status + if (ImportObjectState.EXISTING == pendingDataset.getState()) { + pendingDataset.setState(ImportObjectState.MUTATED); + } + + // Read any observation data stored for these traits + log.debug("fetching observation data stored for traits"); + Set ouDbIds = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(u -> u.getBrAPIObject().getObservationUnitDbId()).collect(Collectors.toSet()); + Set varDbIds = sortedTraits.stream().map(t->t.getObservationVariableDbId()).collect(Collectors.toSet()); + List observations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, varDbIds, program); + + // Construct helper lookup tables to use for hashing stored observation data + Map unitNameByDbId = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, BrAPIObservationUnit::getObservationUnitName)); + Map variableNameByDbId = sortedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableDbId, Trait::getObservationVariableName)); + Map studyNameByDbId = context.getPendingData().getStudyByNameNoScope().values().stream() + .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) + .map(PendingImportObject::getBrAPIObject) + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); + + // Hash stored observation data using a signature of unit, variable, and study names + Map observationByObsHash = observations.stream().collect(Collectors.toMap(o->{ + return observationService.getObservationHash(unitNameByDbId.get(o.getObservationUnitDbId()), + variableNameByDbId.get(o.getObservationVariableDbId()), + studyNameByDbId.get(o.getStudyDbId())); + }, o->o)); + + // Add the observation data map to the context for use in processing import + context.getPendingData().setExistingObsByObsHash(observationByObsHash); + + // Build new pending observation data for each phenotype + Map> pendingObservationByHash = new HashMap<>(); + for (Column column : phenotypeCols) { + + // Checking all import rows for data + for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { + Integer rowNum = i; + ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + String cellData = column.getString(rowNum); + + // Generate hash for looking up prior observation + String unitId = row.getExpUnitId(); + String studyName = context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getStudyName(); + String unitName = context.getExpUnitContext().getPendingObsUnitByOUId().get(unitId).getBrAPIObject().getObservationUnitName(); + String phenoColumnName = column.name(); + String observationHash = observationService.getObservationHash(unitName, phenoColumnName, studyName); + + // Get timestamp if associated column + String timestamp = null; + if ( tsColByPheno.containsKey(phenoColumnName) ) { + timestamp = tsColByPheno.get(phenoColumnName).getString(rowNum); + + // If timestamp is not valid, set to midnight + if (timestamp != null && !timestamp.isBlank() && (!observationService.validDateTimeValue(timestamp) || !observationService.validDateValue(timestamp))) { + timestamp += MIDNIGHT; + + // Add a validation error + ValidationError timestampValErr = new ValidationError(tsColByPheno.get(phenoColumnName).name(), String.format("Timestamp format is not valid for %s", tsColByPheno.get(phenoColumnName).name()), HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(rowNum, timestampValErr); + } + } + + // Construct a pending observation for updates made to prior observations + if (observationByObsHash.containsKey(observationHash)) { + + // Clone the prior observation + BrAPIObservation observation = gson.fromJson(gson.toJson(observationByObsHash.get(observationHash)), BrAPIObservation.class); + + // Construct a pending observation + PendingImportObject pendingObservation = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); + + // Update the pending phenotypic data + if (!cellData.equals(observation.getValue())) { + pendingObservation.getBrAPIObject().setValue(cellData); + pendingObservation.setState(ImportObjectState.MUTATED); + } + + // Update the pending timestamp + if (timestamp != null && !OffsetDateTime.parse(timestamp).equals(observation.getObservationTimeStamp())) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timestamp)); + pendingObservation.getBrAPIObject().setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + pendingObservation.setState(ImportObjectState.MUTATED); + } + + // Add pending observation to map + pendingObservationByHash.put(observationHash, pendingObservation); + + // Construct a pending observation for new data if no prior observation + } else { + UUID trialId = pendingTrial.getId(); + UUID studyId = context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getId(); + String studyYear = context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getSeasons().get(0); + String seasonDbId = studyService.seasonDbIdToYear(studyYear, program.getId()); + + // Clone the observation unit + BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getExpUnitContext().getPendingObsUnitByOUId().get(row.getExpUnitId()).getBrAPIObject()), BrAPIObservationUnit.class); + + // Generate a new ID for the observation + UUID observationID = UUID.randomUUID(); + + // Construct the new observation + BrAPIObservation newObservation = row.constructBrAPIObservation(cellData, phenoColumnName, seasonDbId, observationUnit, context.getImportContext().isCommit(), program, context.getImportContext().getUser(), BRAPI_REFERENCE_SOURCE, trialId, studyId, UUID.fromString(unitId), observationID); + + // Construct a pending observation + PendingImportObject pendingObservation = new PendingImportObject<>(ImportObjectState.NEW, newObservation); + + // Add pending observation to map + pendingObservationByHash.put(observationHash, pendingObservation); + } + } + // Add the pending observation map to the context for use in processing the import + context.getPendingData().setPendingObservationByHash(pendingObservationByHash); + } + } catch (DoesNotExistException | ApiException | UnprocessableEntityException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java index 7e6967bc9..a2536ecf0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java @@ -49,27 +49,24 @@ public boolean process(ExpUnitMiddlewareContext context) { .getAdditionalInfo() .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) .getAsString(); + try { + // Get the dataset belonging to required exp units + dataset = datasetService.fetchDatasetById(datasetId, program).orElseThrow(ApiException::new); + // Construct the pending dataset from the BrAPI observation variable list + pendingDataset = datasetService.constructPIOFromDataset(dataset, program); - // Get the dataset belonging to required exp units - try { - dataset = datasetService.fetchDatasetById(datasetId, program); + // Construct a hashmap to look up the pending dataset by dataset name + pendingDatasetByName = new HashMap<>(); + pendingDatasetByName.put(dataset.getListName(), pendingDataset); + + // Add the map to the context for use in processing import + context.getPendingData().setObsVarDatasetByName(pendingDatasetByName); } catch (ApiException e) { this.compensate(context, new MiddlewareError(() -> { throw new RuntimeException(e); })); } - - // Construct the pending dataset from the BrAPI observation variable list - pendingDataset = datasetService.constructPIOFromDataset(dataset, program); - - // Construct a hashmap to look up the pending dataset by dataset name - pendingDatasetByName = new HashMap<>(); - pendingDatasetByName.put(dataset.getListName(), pendingDataset); - - // Add the map to the context for use in processing import - context.getPendingData().setObsVarDatasetByName(pendingDatasetByName); - return processNext(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java index 7799e9574..28eb14e29 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -28,6 +28,7 @@ public class PendingData { private Map> locationByName; private Map> obsVarDatasetByName; private Map> existingGermplasmByGID; + private Map> pendingObservationByHash; private Map> timeStampColByPheno; private Map existingObsByObsHash; private ValidationErrors validationErrors; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java index a81a5067e..0fc8b807c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java @@ -1,13 +1,18 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.model; import com.fasterxml.jackson.annotation.JsonValue; +import io.micronaut.context.annotation.Property; public class ExpImportProcessConstants { public static final CharSequence COMMA_DELIMITER = ","; public static final String TIMESTAMP_PREFIX = "TS:"; public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; + public static final String MIDNIGHT = "T00:00:00-00:00"; + @Property(name = "brapi.server.reference-source") + public static String BRAPI_REFERENCE_SOURCE; public enum ErrMessage { + MULTIPLE_EXP_TITLES("File contains more than one Experiment Title"), MISSING_OBS_UNIT_ID_ERROR("Experimental entities are missing ObsUnitIDs"), PREEXISTING_EXPERIMENT_TITLE("Experiment Title already exists"); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index 70169478e..6281e74d3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -55,8 +55,8 @@ public DatasetService(BrAPIListDAO brapiListDAO) { * @return BrAPIListDetails object containing the details of the dataset * @throws ApiException if there is an issue with fetching the dataset details from the data source */ - public BrAPIListDetails fetchDatasetById(String id, Program program) throws ApiException { - BrAPIListDetails dataSetDetails = null; + public Optional fetchDatasetById(String id, Program program) throws ApiException { + Optional dataSetDetails = Optional.empty(); // Retrieve existing dataset summaries based on program ID and external reference List existingDatasets = brAPIListDAO @@ -71,9 +71,9 @@ public BrAPIListDetails fetchDatasetById(String id, Program program) throws ApiE } // Retrieve dataset details using the list DB ID from the existing dataset summary - dataSetDetails = brAPIListDAO + dataSetDetails = Optional.ofNullable(brAPIListDAO .getListById(existingDatasets.get(0).getListDbId(), program.getId()) - .getResult(); + .getResult()); return dataSetDetails; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index e421fa54d..67177dad5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -24,10 +24,30 @@ import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.*; import java.util.stream.Collectors; public class ObservationService { + public boolean validDateTimeValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } + + public boolean validDateValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } public String getObservationHash(String observationUnitName, String variableName, String studyName) { String concat = DigestUtils.sha256Hex(observationUnitName) + DigestUtils.sha256Hex(variableName) + diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index 53015159b..7114e0b04 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -38,7 +38,7 @@ @Singleton @Slf4j public class StudyService { - + private final Map seasonDbIdToYearCache = new HashMap<>(); private final BrAPISeasonDAO brAPISeasonDAO; private final BrAPIStudyDAO brAPIStudyDAO; @@ -52,7 +52,6 @@ public StudyService(BrAPISeasonDAO brAPISeasonDAO, this.brAPIStudyDAO = brAPIStudyDAO; } - // TODO: used by both workflows public PendingImportObject processAndCacheStudy( BrAPIStudy existingStudy, @@ -111,7 +110,7 @@ public PendingImportObject constructPIOFromBrapiStudy(BrAPIStudy brA } // TODO: used by both workflows - private String seasonDbIdToYear(String seasonDbId, UUID programId) { + public String seasonDbIdToYear(String seasonDbId, UUID programId) { String year = null; // TODO: add season objects to redis cache then just extract year from those // removing this for now here From d6ed109340eb7006043794b26778240cd961131a Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 17 May 2024 14:20:04 -0400 Subject: [PATCH 056/203] cosntruct mapped pending import rows --- .../middleware/process/DataValidation.java | 65 -------------- .../process/ImportPreviewStatistics.java | 5 +- .../middleware/process/ValidationPrep.java | 84 +++++++++++++++++++ .../process/brapi/PendingObservation.java | 58 +++++++++---- .../experiment/service/DatasetService.java | 2 +- .../service/ObservationUnitService.java | 19 ----- .../experiment/service/StudyService.java | 2 +- .../experiment/service/TrialService.java | 5 +- 8 files changed, 132 insertions(+), 108 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/DataValidation.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ValidationPrep.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/DataValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/DataValidation.java deleted file mode 100644 index 65b4c7998..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/DataValidation.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; - -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; -import org.breedinginsight.model.Program; - -import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; - -@Slf4j -public class DataValidation extends ExpUnitMiddleware { - ObservationUnitService observationUnitService; - - @Inject - public DataValidation(ObservationUnitService observationUnitService) { - this.observationUnitService = observationUnitService; - } - - @Override - public boolean process(ExpUnitMiddlewareContext context) { - Program program; - Set expUnitIds; - List missingIds; - List brapiUnits; - List> pendingUnits; - Map> pendingUnitById; - Map> pendingUnitByNameNoScope; - - log.debug("fetching required exp units from BrAPI service"); - program = context.getImportContext().getProgram(); - - // Collect deltabreed-generated exp unit ids listed in the import - expUnitIds = context.getExpUnitContext().getReferenceOUIds(); - try { - // For each id fetch the observation unit from the brapi data store - brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); - - // Construct pending import objects from the units - pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); - - // Construct a hashmap to look up the pending unit by ID - pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); - - // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed - pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); - - // add maps to the context for use in processing import - context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); - context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); - } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); - } - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java index 0ead6a47e..3f35afea5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java @@ -3,7 +3,6 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi.NewPendingBrAPIObjects; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi.PendingObservation; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import javax.inject.Inject; @@ -13,12 +12,12 @@ public class ImportPreviewStatistics extends ExpUnitMiddleware { ExpUnitMiddleware middleware; private Provider newPendingBrAPIObjectsProvider; - private Provider dataValidationProvider; + private Provider dataValidationProvider; private Provider fieldValidationProvider; @Inject public ImportPreviewStatistics(Provider newPendingBrAPIObjectsProvider, - Provider dataValidationProvider, + Provider dataValidationProvider, Provider fieldValidationProvider) { this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ValidationPrep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ValidationPrep.java new file mode 100644 index 000000000..72fecfd21 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ValidationPrep.java @@ -0,0 +1,84 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import tech.tablesaw.api.Table; +import tech.tablesaw.columns.Column; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.ObservationProcessor.getObservationHash; + +@Slf4j +public class ValidationPrep extends ExpUnitMiddleware { + ObservationService observationService; + + @Inject + public ValidationPrep(ObservationService observationService) { + this.observationService = observationService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + // Get all the dynamic columns of the import + ImportUpload upload = context.getImportContext().getUpload(); + Table data = context.getImportContext().getData(); + String[] dynamicColNames = upload.getDynamicColumnNames(); + List> dynamicCols = data.columns(dynamicColNames); + + // Collect the columns for observation variable data + List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); + Set varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toSet()); + + for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { + Integer rowNum = i; + ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + + // Fetch the pending import for the row, creating one if null + PendingImport mappedImportRow = context.getImportContext().getMappedBrAPIImport().getOrDefault(rowNum, new PendingImport()); + + List> observations = mappedImportRow.getObservations(); + + String unitId = row.getObsUnitID(); + mappedImportRow.setTrial(context.getExpUnitContext().getPendingTrialByOUId().get(unitId)); + mappedImportRow.setLocation(context.getExpUnitContext().getPendingLocationByOUId().get(unitId)); + mappedImportRow.setStudy(context.getExpUnitContext().getPendingStudyByOUId().get(unitId)); + mappedImportRow.setObservationUnit(context.getExpUnitContext().getPendingObsUnitByOUId().get(unitId)); + mappedImportRow.setGermplasm(context.getExpUnitContext().getPendingGermplasmByOUId().get(unitId)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + String observationHash = observationService.getObservationHash( + pendingStudyByOUId.get(unitId).getBrAPIObject().getStudyName() + + pendingObsUnitByOUId.get(unitId).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(column), + pendingStudyByOUId.get(unitId).getBrAPIObject().getStudyName() + ); + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(observationHash)); + } + } + +// try { +// +// } catch (ApiException e) { +// this.compensate(context, new MiddlewareError(() -> { +// throw new RuntimeException(e); +// })); +// } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java index 162ee0842..e67ec1b04 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java @@ -16,6 +16,7 @@ import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -161,16 +162,27 @@ public boolean process(ExpUnitMiddlewareContext context) { // Build new pending observation data for each phenotype Map> pendingObservationByHash = new HashMap<>(); - for (Column column : phenotypeCols) { - // Checking all import rows for data - for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { - Integer rowNum = i; - ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + + // Checking all import rows for data + for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { + Integer rowNum = i; + ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + + // Construct the pending import for the row + PendingImport mappedImportRow = context.getImportContext().getMappedBrAPIImport().getOrDefault(rowNum, new PendingImport()); + String unitId = row.getObsUnitID(); + mappedImportRow.setTrial(context.getExpUnitContext().getPendingTrialByOUId().get(unitId)); + mappedImportRow.setLocation(context.getExpUnitContext().getPendingLocationByOUId().get(unitId)); + mappedImportRow.setStudy(context.getExpUnitContext().getPendingStudyByOUId().get(unitId)); + mappedImportRow.setObservationUnit(context.getExpUnitContext().getPendingObsUnitByOUId().get(unitId)); + mappedImportRow.setGermplasm(context.getExpUnitContext().getPendingGermplasmByOUId().get(unitId)); + + // For each phenotype, construct the pending observations + for (Column column : phenotypeCols) { String cellData = column.getString(rowNum); // Generate hash for looking up prior observation - String unitId = row.getExpUnitId(); String studyName = context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getStudyName(); String unitName = context.getExpUnitContext().getPendingObsUnitByOUId().get(unitId).getBrAPIObject().getObservationUnitName(); String phenoColumnName = column.name(); @@ -178,7 +190,7 @@ public boolean process(ExpUnitMiddlewareContext context) { // Get timestamp if associated column String timestamp = null; - if ( tsColByPheno.containsKey(phenoColumnName) ) { + if (tsColByPheno.containsKey(phenoColumnName)) { timestamp = tsColByPheno.get(phenoColumnName).getString(rowNum); // If timestamp is not valid, set to midnight @@ -191,31 +203,34 @@ public boolean process(ExpUnitMiddlewareContext context) { } } - // Construct a pending observation for updates made to prior observations + // Construct a pending observation if this is a prior observation if (observationByObsHash.containsKey(observationHash)) { // Clone the prior observation BrAPIObservation observation = gson.fromJson(gson.toJson(observationByObsHash.get(observationHash)), BrAPIObservation.class); // Construct a pending observation - PendingImportObject pendingObservation = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); + PendingImportObject pendingPriorObservation = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); // Update the pending phenotypic data if (!cellData.equals(observation.getValue())) { - pendingObservation.getBrAPIObject().setValue(cellData); - pendingObservation.setState(ImportObjectState.MUTATED); + pendingPriorObservation.getBrAPIObject().setValue(cellData); + pendingPriorObservation.setState(ImportObjectState.MUTATED); } // Update the pending timestamp if (timestamp != null && !OffsetDateTime.parse(timestamp).equals(observation.getObservationTimeStamp())) { DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timestamp)); - pendingObservation.getBrAPIObject().setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); - pendingObservation.setState(ImportObjectState.MUTATED); + pendingPriorObservation.getBrAPIObject().setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + pendingPriorObservation.setState(ImportObjectState.MUTATED); } + // Set the pending prior observation in the pending import for the row + mappedImportRow.getObservations().add(pendingPriorObservation); + // Add pending observation to map - pendingObservationByHash.put(observationHash, pendingObservation); + pendingObservationByHash.put(observationHash, pendingPriorObservation); // Construct a pending observation for new data if no prior observation } else { @@ -234,15 +249,22 @@ public boolean process(ExpUnitMiddlewareContext context) { BrAPIObservation newObservation = row.constructBrAPIObservation(cellData, phenoColumnName, seasonDbId, observationUnit, context.getImportContext().isCommit(), program, context.getImportContext().getUser(), BRAPI_REFERENCE_SOURCE, trialId, studyId, UUID.fromString(unitId), observationID); // Construct a pending observation - PendingImportObject pendingObservation = new PendingImportObject<>(ImportObjectState.NEW, newObservation); + PendingImportObject pendingNewObservation = new PendingImportObject<>(ImportObjectState.NEW, newObservation); + + // Set the new pending observation in the pending import for the row + mappedImportRow.getObservations().add(pendingNewObservation); // Add pending observation to map - pendingObservationByHash.put(observationHash, pendingObservation); + pendingObservationByHash.put(observationHash, pendingNewObservation); } } - // Add the pending observation map to the context for use in processing the import - context.getPendingData().setPendingObservationByHash(pendingObservationByHash); + + // Set the pending import for the row + context.getImportContext().getMappedBrAPIImport().put(rowNum, mappedImportRow); } + + // Add the pending observation map to the context for use in processing the import + context.getPendingData().setPendingObservationByHash(pendingObservationByHash); } catch (DoesNotExistException | ApiException | UnprocessableEntityException e) { this.compensate(context, new MiddlewareError(() -> { throw new RuntimeException(e); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index 6281e74d3..399f23468 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -93,7 +93,7 @@ public PendingImportObject constructPIOFromDataset(BrAPIListDe // Get the external reference for the dataset from the existing list BrAPIExternalReference xref = Utilities.getExternalReference(dataset.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) - .orElseThrow(() -> new IllegalStateException("External references weren't found for list (dbid): " + dataset.getListDbId()); + .orElseThrow(() -> new IllegalStateException("External references weren't found for list (dbid): " + dataset.getListDbId())); // Create a PendingImportObject for the dataset with the existing list and reference ID return new PendingImportObject(ImportObjectState.EXISTING, dataset, UUID.fromString(xref.getReferenceId())); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index e19c196d1..dd8bc2856 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -351,25 +351,6 @@ public List commitUpdatedPendingObservationUnitToBrAPIStor return updatedUnits; } - private void updateObsUnitDependencyValues(String programKey) { - - // update study DbIds - this.studyByNameNoScope.values() - .stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(study -> updateStudyDbId(study, programKey)); - - // update germplasm DbIds - this.existingGermplasmByGID.values() - .stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(this::updateGermplasmDbId); - } - private void updateStudyDbId(BrAPIStudy study, String programKey) { this.observationUnitByNameNoScope.values() .stream() diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index 7114e0b04..f6679e947 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -92,7 +92,7 @@ public PendingImportObject constructPIOFromBrapiStudy(BrAPIStudy brA // Retrieve external reference for the study BrAPIExternalReference xref = Utilities.getExternalReference(brAPIStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) - .orElseThrow(() -> new IllegalStateException("External references weren't found for study (dbid): " + brAPIStudy.getStudyDbId()); + .orElseThrow(() -> new IllegalStateException("External references weren't found for study (dbid): " + brAPIStudy.getStudyDbId())); // Map season dbid to year String seasonDbId = brAPIStudy.getSeasons().get(0); // It is assumed that the study has only one season diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index 430153d40..b82c5169d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -298,7 +298,7 @@ public PendingImportObject constructPIOFromBrapiTrial(BrAPITrial tri String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); - pio = new PendingImportObject<>(ImportObjectState.EXISTING, trial, experimentId)); + pio = new PendingImportObject<>(ImportObjectState.EXISTING, trial, experimentId); return pio; } @@ -335,6 +335,9 @@ private Map> initializeTrialByNameNoScop return trialByName; } + private void initializeTrialsForExistingObservationUnits(Program program, Map> observationUnitByNameNoScope, Map> trialByName) { + } + // TODO: used by expunit workflow public Map> mapPendingTrialByOUId( String unitId, From 7cb1f589f9bd3693744154ea66eaa47edfc683af Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 17 May 2024 18:20:01 -0400 Subject: [PATCH 057/203] validate and record changelog --- .../middleware/process/FieldValidation.java | 28 +------ .../process/ImportPreviewStatistics.java | 3 - .../middleware/process/ValidationPrep.java | 84 ------------------- .../process/brapi/PendingObservation.java | 53 +++++++++++- 4 files changed, 52 insertions(+), 116 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ValidationPrep.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java index d27190547..d28a59e2b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java @@ -25,36 +25,10 @@ public FieldValidation(ObservationUnitService observationUnitService) { @Override public boolean process(ExpUnitMiddlewareContext context) { - Program program; - Set expUnitIds; - List missingIds; - List brapiUnits; - List> pendingUnits; - Map> pendingUnitById; - Map> pendingUnitByNameNoScope; - log.debug("fetching required exp units from BrAPI service"); - program = context.getImportContext().getProgram(); - - // Collect deltabreed-generated exp unit ids listed in the import - expUnitIds = context.getExpUnitContext().getReferenceOUIds(); try { - // For each id fetch the observation unit from the brapi data store - brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); - - // Construct pending import objects from the units - pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); - - // Construct a hashmap to look up the pending unit by ID - pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); - - // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed - pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); - // add maps to the context for use in processing import - context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); - context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); - } catch (ApiException e) { + } catch (Exception e) { this.compensate(context, new MiddlewareError(() -> { throw new RuntimeException(e); })); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java index 3f35afea5..8e5a10db7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java @@ -12,17 +12,14 @@ public class ImportPreviewStatistics extends ExpUnitMiddleware { ExpUnitMiddleware middleware; private Provider newPendingBrAPIObjectsProvider; - private Provider dataValidationProvider; private Provider fieldValidationProvider; @Inject public ImportPreviewStatistics(Provider newPendingBrAPIObjectsProvider, - Provider dataValidationProvider, Provider fieldValidationProvider) { this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( newPendingBrAPIObjectsProvider.get(), // Construct Pending import objects for new BrAPI data - dataValidationProvider.get(), // Validate data fieldValidationProvider.get()); // Validate fields } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ValidationPrep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ValidationPrep.java deleted file mode 100644 index 72fecfd21..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ValidationPrep.java +++ /dev/null @@ -1,84 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; - -import lombok.extern.slf4j.Slf4j; -import org.brapi.v2.model.pheno.BrAPIObservation; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.PendingImport; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; -import tech.tablesaw.api.Table; -import tech.tablesaw.columns.Column; - -import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; - -import static org.breedinginsight.brapps.importer.services.processors.ObservationProcessor.getObservationHash; - -@Slf4j -public class ValidationPrep extends ExpUnitMiddleware { - ObservationService observationService; - - @Inject - public ValidationPrep(ObservationService observationService) { - this.observationService = observationService; - } - - @Override - public boolean process(ExpUnitMiddlewareContext context) { - // Get all the dynamic columns of the import - ImportUpload upload = context.getImportContext().getUpload(); - Table data = context.getImportContext().getData(); - String[] dynamicColNames = upload.getDynamicColumnNames(); - List> dynamicCols = data.columns(dynamicColNames); - - // Collect the columns for observation variable data - List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); - Set varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toSet()); - - for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { - Integer rowNum = i; - ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); - - // Fetch the pending import for the row, creating one if null - PendingImport mappedImportRow = context.getImportContext().getMappedBrAPIImport().getOrDefault(rowNum, new PendingImport()); - - List> observations = mappedImportRow.getObservations(); - - String unitId = row.getObsUnitID(); - mappedImportRow.setTrial(context.getExpUnitContext().getPendingTrialByOUId().get(unitId)); - mappedImportRow.setLocation(context.getExpUnitContext().getPendingLocationByOUId().get(unitId)); - mappedImportRow.setStudy(context.getExpUnitContext().getPendingStudyByOUId().get(unitId)); - mappedImportRow.setObservationUnit(context.getExpUnitContext().getPendingObsUnitByOUId().get(unitId)); - mappedImportRow.setGermplasm(context.getExpUnitContext().getPendingGermplasmByOUId().get(unitId)); - - // loop over phenotype column observation data for current row - for (Column column : phenotypeCols) { - String observationHash = observationService.getObservationHash( - pendingStudyByOUId.get(unitId).getBrAPIObject().getStudyName() + - pendingObsUnitByOUId.get(unitId).getBrAPIObject().getObservationUnitName(), - getVariableNameFromColumn(column), - pendingStudyByOUId.get(unitId).getBrAPIObject().getStudyName() - ); - - // if value was blank won't be entry in map for this observation - observations.add(observationByHash.get(observationHash)); - } - } - -// try { -// -// } catch (ApiException e) { -// this.compensate(context, new MiddlewareError(() -> { -// throw new RuntimeException(e); -// })); -// } - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java index e67ec1b04..6413e6c59 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java @@ -1,6 +1,8 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi; import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -16,6 +18,7 @@ import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -35,6 +38,7 @@ import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; +import org.jooq.impl.QOM; import tech.tablesaw.api.Table; import tech.tablesaw.columns.Column; @@ -82,11 +86,11 @@ public boolean process(ExpUnitMiddlewareContext context) { List> dynamicCols = data.columns(dynamicColNames); // Collect the columns for observation variable data - List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); + List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); Set varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toSet()); // Collect the columns for observation timestamps - List> timestampCols = dynamicCols.stream().filter(col -> col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); + List> timestampCols = dynamicCols.stream().filter(col -> col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); Set tsNames = timestampCols.stream().map(Column::name).collect(Collectors.toSet()); // Construct validation errors for any timestamp columns that don't have a matching variable column @@ -212,10 +216,23 @@ public boolean process(ExpUnitMiddlewareContext context) { // Construct a pending observation PendingImportObject pendingPriorObservation = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); + // Are overwrites authorized? + boolean canOverwrite = context.getImportContext().isCommit() && "false".equals( row.getOverwrite() == null ? "false" : row.getOverwrite()); + String original = null; + // Update the pending phenotypic data if (!cellData.equals(observation.getValue())) { pendingPriorObservation.getBrAPIObject().setValue(cellData); pendingPriorObservation.setState(ImportObjectState.MUTATED); + + // Validation error if user has not chosen to overwrite existing data + if (StringUtils.isNotBlank(cellData) && !canOverwrite) { + ValidationError overwriteValErr = new ValidationError(tsColByPheno.get(phenoColumnName).name(), String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", unitId, phenoColumnName), HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(rowNum, overwriteValErr); + } + + // Record original value in changelog entry + original = observation.getValue(); } // Update the pending timestamp @@ -224,6 +241,38 @@ public boolean process(ExpUnitMiddlewareContext context) { String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timestamp)); pendingPriorObservation.getBrAPIObject().setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); pendingPriorObservation.setState(ImportObjectState.MUTATED); + + // Validation error if user has not chosen to overwrite existing timestamp + if (StringUtils.isNotBlank(cellData) && !canOverwrite) { + ValidationError overwriteValErr = new ValidationError(tsColByPheno.get(phenoColumnName).name(), String.format("Value already exists for ObsUnitId: %s, Timestamp: %s", unitId, tsColByPheno.get(phenoColumnName).name()), HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(rowNum, overwriteValErr); + } + + // Add original timestamp to changelog entry + original = Optional.ofNullable(original).map(o -> o + " " + observation.getObservationTimeStamp()).orElse(String.valueOf(observation.getObservationTimeStamp())); + + } + + // Record any updates as BrAPI observation additional info + if (context.getImportContext().isCommit()) { + + // Create the changelog field in observation additional info if it does not already exist + if (pendingPriorObservation.getBrAPIObject().getAdditionalInfo().isJsonNull()) { + pendingPriorObservation.getBrAPIObject().setAdditionalInfo(new JsonObject()); + pendingPriorObservation.getBrAPIObject().getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + if (pendingPriorObservation.getBrAPIObject().getAdditionalInfo() != null && !pendingPriorObservation.getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { + pendingPriorObservation.getBrAPIObject().getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + + // Construct a changelog entry + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String rightNow = formatter.format(OffsetDateTime.now()); + String reason = Optional.ofNullable(row.getOverwriteReason()).orElse(""); + ChangeLogEntry entry = new ChangeLogEntry(original, reason, context.getImportContext().getUser().getId(), rightNow); + + // Add the entry to the changelog + pendingPriorObservation.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(entry).getAsJsonObject()); } // Set the pending prior observation in the pending import for the row From 0585568cda37b165ae4fddf5dd79635c1d75527e Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 20 May 2024 11:21:01 -0400 Subject: [PATCH 058/203] Moved create workflow work to own branch and rearranged things a little --- .../importer/services/pipeline/Pipeline.java | 35 ++ .../services/pipeline/ProcessingStep.java | 23 + .../experiment/ExperimentUtilities.java | 18 + .../experiment/create/model/PendingData.java | 27 + .../workflow/CreateNewExperimentWorkflow.java | 25 +- .../steps/GetExistingProcessingStep.java | 470 ++++++++++++++++++ .../create/workflow/steps/ProcessStep.java | 15 + .../services/SharedStudyService.java | 120 +++++ .../services/SharedTrialService.java | 180 +++++++ 9 files changed, 912 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/pipeline/Pipeline.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/Pipeline.java b/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/Pipeline.java new file mode 100644 index 000000000..76c4433ee --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/Pipeline.java @@ -0,0 +1,35 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.pipeline; + +public class Pipeline { + + private final ProcessingStep currentStep; + + public Pipeline(ProcessingStep currentStep) { + this.currentStep = currentStep; + } + + public Pipeline addProcessingStep(ProcessingStep newStep) { + return new Pipeline<>(input -> newStep.process(currentStep.process(input))); + } + + public O execute(I input) { + return currentStep.process(input); + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java new file mode 100644 index 000000000..ca16792db --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java @@ -0,0 +1,23 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.breedinginsight.brapps.importer.services.pipeline; + +public interface ProcessingStep { + O process(I input); +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java new file mode 100644 index 000000000..ed685449d --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -0,0 +1,18 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment; + +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; + +import java.util.*; +import java.util.stream.Collectors; + +public class ExperimentUtilities { + + public static final CharSequence COMMA_DELIMITER = ","; + + public static List importRowsToExperimentObservations(List importRows) { + return importRows.stream() + .map(trialImport -> (ExperimentObservation) trialImport) + .collect(Collectors.toList()); + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java new file mode 100644 index 000000000..7d6fc74b9 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -0,0 +1,27 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; + +import lombok.*; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.ProgramLocation; + +import java.util.Map; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class PendingData { + private Map> observationUnitByNameNoScope; + private Map> trialByNameNoScope; + private Map> studyByNameNoScope; + private Map> locationByName; + private Map> obsVarDatasetByName; + private Map> existingGermplasmByGID; +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 6f6ab661a..ebfa5469d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -21,21 +21,42 @@ import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.pipeline.Pipeline; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.GetExistingProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.ProcessStep; +import javax.inject.Inject; import javax.inject.Named; +import javax.inject.Provider; /** * This class represents a workflow for creating a new experiment. The bean name must match the appropriate bean column * value in the import_mapping_workflow db table */ + @Prototype @Named("CreateNewExperimentWorkflow") public class CreateNewExperimentWorkflow implements Workflow { + private final Provider getExistingStepProvider; + private final Provider processStepProvider; + + @Inject + public CreateNewExperimentWorkflow(Provider getExistingStepProvider, + Provider processStepProvider) { + this.getExistingStepProvider = getExistingStepProvider; + this.processStepProvider = processStepProvider; + } + @Override public ProcessedData process(ImportContext context) { // TODO - return null; + Pipeline pipeline = new Pipeline<>(getExistingStepProvider.get()) + .addProcessingStep(processStepProvider.get()); + ProcessedData processed = pipeline.execute(context); + + // TODO: return actual data + return processed; } /** @@ -43,8 +64,10 @@ public ProcessedData process(ImportContext context) { * * @return the name of the workflow */ + @Override public String getName() { return "CreateNewExperimentWorkflow"; } } + diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java new file mode 100644 index 000000000..10afb13a8 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java @@ -0,0 +1,470 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; + +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.BrAPIListTypes; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.*; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedStudyService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedTrialService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.ProgramLocationService; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +/** + * References code common between workflows in shared services. DAO access is done directly in the + * steps rather than another layer of services. + */ + +@Prototype +@Slf4j +public class GetExistingProcessingStep implements ProcessingStep { + + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private final BrAPITrialDAO brAPITrialDAO; + private final BrAPIStudyDAO brAPIStudyDAO; + private final ProgramLocationService locationService; + private final BrAPIListDAO brAPIListDAO; + private final BrAPIGermplasmDAO brAPIGermplasmDAO; + private final SharedStudyService sharedStudyService; + private final SharedTrialService sharedTrialService; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public GetExistingProcessingStep(BrAPIObservationUnitDAO brAPIObservationUnitDAO, + BrAPITrialDAO brAPITrialDAO, + BrAPIStudyDAO brAPIStudyDAO, + ProgramLocationService locationService, + BrAPIListDAO brAPIListDAO, + BrAPIGermplasmDAO brAPIGermplasmDAO, + SharedStudyService sharedStudyService, + SharedTrialService sharedTrialService) { + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + this.brAPITrialDAO = brAPITrialDAO; + this.brAPIStudyDAO = brAPIStudyDAO; + this.locationService = locationService; + this.brAPIListDAO = brAPIListDAO; + this.brAPIGermplasmDAO = brAPIGermplasmDAO; + this.sharedStudyService = sharedStudyService; + this.sharedTrialService = sharedTrialService; + } + + @Override + public PendingData process(ImportContext input) { + + List experimentImportRows = ExperimentUtilities.importRowsToExperimentObservations(input.getImportRows()); + Program program = input.getProgram(); + + // Populate pending objects with existing status + Map> observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); + Map> trialByNameNoScope = sharedTrialService.initializeTrialByNameNoScope(program, observationUnitByNameNoScope, experimentImportRows); + Map> studyByNameNoScope = initializeStudyByNameNoScope(program, trialByNameNoScope, observationUnitByNameNoScope, experimentImportRows); + // interesting we're using our data model instead of brapi for locations + Map> locationByName = initializeUniqueLocationNames(program, studyByNameNoScope, experimentImportRows); + Map> obsVarDatasetByName = initializeObsVarDatasetByName(program, trialByNameNoScope, experimentImportRows); + Map> existingGermplasmByGID = initializeExistingGermplasmByGID(program, observationUnitByNameNoScope, experimentImportRows); + + PendingData existing = PendingData.builder() + .observationUnitByNameNoScope(observationUnitByNameNoScope) + .trialByNameNoScope(trialByNameNoScope) + .studyByNameNoScope(studyByNameNoScope) + .locationByName(locationByName) + .obsVarDatasetByName(obsVarDatasetByName) + .existingGermplasmByGID(existingGermplasmByGID) + .build(); + + return existing; + } + + /** + * Initializes the observation units for the given program and experimentImportRows. + * + * @param program The program object + * @param experimentImportRows A list of ExperimentObservation objects + * @return A map of Observation Unit IDs to PendingImportObject objects + * + * @throws InternalServerException + * @throws IllegalStateException + */ + private Map> initializeObservationUnits(Program program, List experimentImportRows) { + Map> observationUnitByName = new HashMap<>(); + + Map rowByObsUnitId = new HashMap<>(); + experimentImportRows.forEach(row -> { + if (StringUtils.isNotBlank(row.getObsUnitID())) { + if(rowByObsUnitId.containsKey(row.getObsUnitID())) { + throw new IllegalStateException("ObsUnitId is repeated: " + row.getObsUnitID()); + } + rowByObsUnitId.put(row.getObsUnitID(), row); + } + }); + + try { + List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); + + // TODO: grab from externalReferences + /* + observationUnitByObsUnitId = existingObsUnits.stream() + .collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, + (BrAPIObservationUnit unit) -> new PendingImportObject<>(unit, false))); + */ + + String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + if (existingObsUnits.size() == rowByObsUnitId.size()) { + existingObsUnits.forEach(brAPIObservationUnit -> { + processAndCacheObservationUnit(brAPIObservationUnit, refSource, program, observationUnitByName, rowByObsUnitId); + + BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) + .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); + + ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); + row.setExpTitle(Utilities.removeProgramKey(brAPIObservationUnit.getTrialName(), program.getKey())); + row.setEnv(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getStudyName(), program.getKey())); + row.setEnvLocation(Utilities.removeProgramKey(brAPIObservationUnit.getLocationName(), program.getKey())); + }); + } else { + List missingIds = new ArrayList<>(rowByObsUnitId.keySet()); + missingIds.removeAll(existingObsUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList())); + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + + return observationUnitByName; + } catch (ApiException e) { + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + /** + * Adds a new map entry to observationUnitByName based on the brAPIObservationUnit passed in and sets the + * expUnitId in the rowsByObsUnitId map. + * + * @param brAPIObservationUnit the BrAPI observation unit object + * @param refSource the reference source + * @param program the program object + * @param observationUnitByName the map of observation units by name (will be modified in place) + * @param rowByObsUnitId the map of rows by observation unit ID (will be modified in place) + * + * @throws InternalServerException + */ + private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservationUnit, String refSource, Program program, + Map> observationUnitByName, + Map rowByObsUnitId) { + BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) + .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); + + ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); + row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); + observationUnitByName.put(createObservationUnitKey(row), + new PendingImportObject<>(ImportObjectState.EXISTING, + brAPIObservationUnit, + UUID.fromString(idRef.getReferenceId()))); + } + + private String createObservationUnitKey(ExperimentObservation importRow) { + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); + } + + private String createObservationUnitKey(String studyName, String obsUnitName) { + return studyName + obsUnitName; + } + + /** + * Initializes studies by name without scope. + * + * @param program The program object. + * @param trialByNameNoScope A map of trial names with their corresponding pending import objects. + * @param experimentImportRows A list of experiment observation objects. + * @return A map of study names with their corresponding pending import objects. + * @throws InternalServerException If there is an error while processing the method. + */ + private Map> initializeStudyByNameNoScope(Program program, + Map> trialByNameNoScope, + Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> studyByName = new HashMap<>(); + if (trialByNameNoScope.size() != 1) { + return studyByName; + } + + try { + initializeStudiesForExistingObservationUnits(program, studyByName, observationUnitByNameNoScope); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing studies", e); + throw new InternalServerException(e.toString(), e); + } + + List existingStudies; + Optional> trial = getTrialPIO(experimentImportRows, trialByNameNoScope); + + try { + if (trial.isEmpty()) { + // TODO: throw ValidatorException and return 422 + } + UUID experimentId = trial.get().getId(); + existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); + for (BrAPIStudy existingStudy : existingStudies) { + sharedStudyService.processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); + } + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing studies: ", e); + throw new InternalServerException(e.toString(), e); + } + + return studyByName; + } + + /** + * Retrieves the PendingImportObject of a BrAPITrial based on the given list of ExperimentObservation and trialByNameNoScope map. + * + * @param experimentImportRows The list of ExperimentObservation objects. + * @param trialByNameNoScope The map of trial names to PendingImportObject of BrAPITrial. + * @return The Optional containing the PendingImportObject of BrAPITrial, or an empty Optional if no matching trial is found. + */ + private Optional> getTrialPIO(List experimentImportRows, + Map> trialByNameNoScope) { + Optional expTitle = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID()) && StringUtils.isNotBlank(row.getExpTitle())) + .map(ExperimentObservation::getExpTitle) + .findFirst(); + + if (expTitle.isEmpty() && trialByNameNoScope.keySet().stream().findFirst().isEmpty()) { + return Optional.empty(); + } + if(expTitle.isEmpty()) { + expTitle = trialByNameNoScope.keySet().stream().findFirst(); + } + + return Optional.ofNullable(trialByNameNoScope.get(expTitle.get())); + } + + + private void initializeStudiesForExistingObservationUnits( + Program program, + Map> studyByName, + Map> observationUnitByNameNoScope + ) throws Exception { + Set studyDbIds = observationUnitByNameNoScope.values() + .stream() + .map(pio -> pio.getBrAPIObject() + .getStudyDbId()) + .collect(Collectors.toSet()); + + List studies = sharedStudyService.fetchStudiesByDbId(studyDbIds, program); + for (BrAPIStudy study : studies) { + sharedStudyService.processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); + } + } + + /** + * Initializes unique location names for a program. + * + * @param program The program object. + * @param studyByNameNoScope A map of study names and corresponding BrAPI study objects. + * @param experimentImportRows A list of experiment observation objects for import. + * @return A map of location names and their corresponding pending import objects. + * @throws InternalServerException If there is an error fetching locations. + */ + private Map> initializeUniqueLocationNames(Program program, + Map> studyByNameNoScope, + List experimentImportRows) { + Map> locationByName = new HashMap<>(); + + List existingLocations = new ArrayList<>(); + if(studyByNameNoScope.size() > 0) { + Set locationDbIds = studyByNameNoScope.values() + .stream() + .map(study -> study.getBrAPIObject() + .getLocationDbId()) + .collect(Collectors.toSet()); + try { + existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueLocationNames = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getEnvLocation) + .distinct() + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + try { + existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); + return locationByName; + } + + /** + * Initializes observation variable dataset by name. + * + * @param program The program associated with the dataset. + * @param trialByNameNoScope The map of trials identified by name without scope. + * @param experimentImportRows The list of experiment observation rows. + * @return The map of observation variable dataset indexed by name. + * + * @throws InternalServerException + */ + private Map> initializeObsVarDatasetByName(Program program, + Map> trialByNameNoScope, + List experimentImportRows) { + Map> obsVarDatasetByName = new HashMap<>(); + + Optional> trialPIO = getTrialPIO(experimentImportRows, trialByNameNoScope); + + if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String datasetId = trialPIO.get().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + try { + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(datasetId)); + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("existing dataset summary not returned from brapi server"); + } + BrAPIListDetails dataSetDetails = brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult(); + processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + return obsVarDatasetByName; + } + + /** + * Process and cache an object of type BrAPIListDetails. + * + * @param existingList The existing list to be processed and cached + * @param obsVarDatasetByName A map of ObsVarDatasets indexed by name (will be modified in place) + * + * @throws IllegalStateException + */ + private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map> obsVarDatasetByName) { + BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for list (dbid): " + existingList.getListDbId())); + obsVarDatasetByName.put(existingList.getListName(), + new PendingImportObject<>(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceId()))); + } + + /** + * Initializes existing germplasm objects by germplasm ID (GID). + * + * @param program The program object. + * @param observationUnitByNameNoScope A map of observation unit objects by name. + * @param experimentImportRows A list of experiment observation objects. + * @return A map of existing germplasm objects by germplasm ID. + * + * @throws InternalServerException + */ + private Map> initializeExistingGermplasmByGID(Program program, + Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> existingGermplasmByGID = new HashMap<>(); + + List existingGermplasms = new ArrayList<>(); + if(observationUnitByNameNoScope.size() > 0) { + Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + try { + existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueGermplasmGIDs = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getGid) + .distinct() + .collect(Collectors.toList()); + + try { + existingGermplasms.addAll(getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + existingGermplasms.forEach(existingGermplasm -> { + BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); + }); + return existingGermplasmByGID; + } + + /** + * Retrieves a list of germplasm with the specified accession numbers. + * + * @param germplasmAccessionNumbers The list of accession numbers to search for. + * @param programId The ID of the program. + * @return An ArrayList of BrAPIGermplasm objects that match the accession numbers. + * @throws ApiException if there is an error retrieving the germplasm. + */ + private ArrayList getGermplasmByAccessionNumber( + List germplasmAccessionNumbers, + UUID programId) throws ApiException { + List germplasmList = brAPIGermplasmDAO.getGermplasm(programId); + ArrayList resultGermplasm = new ArrayList<>(); + // Search for accession number matches + for (BrAPIGermplasm germplasm : germplasmList) { + for (String accessionNumber : germplasmAccessionNumbers) { + if (germplasm.getAccessionNumber() + .equals(accessionNumber)) { + resultGermplasm.add(germplasm); + break; + } + } + } + return resultGermplasm; + } + +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java new file mode 100644 index 000000000..67127c103 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java @@ -0,0 +1,15 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; + +import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; + +public class ProcessStep implements ProcessingStep { + + @Override + public ProcessedData process(PendingData input) { + + // TODO: implement + return new ProcessedData(); + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java new file mode 100644 index 000000000..80af3e2a2 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java @@ -0,0 +1,120 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import io.micronaut.context.annotation.Property; +import io.reactivex.functions.Function; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPISeason; +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +@Singleton +@Slf4j +public class SharedStudyService { + + private final BrAPISeasonDAO brAPISeasonDAO; + private final BrAPIStudyDAO brAPIStudyDAO; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public SharedStudyService(BrAPISeasonDAO brAPISeasonDAO, + BrAPIStudyDAO brAPIStudyDAO) { + this.brAPISeasonDAO = brAPISeasonDAO; + this.brAPIStudyDAO = brAPIStudyDAO; + } + + // TODO: used by both workflows + public PendingImportObject processAndCacheStudy( + BrAPIStudy existingStudy, + Program program, + Function getterFunction, + Map> studyMap) throws Exception { + PendingImportObject pendingStudy; + BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); + // map season dbid to year + String seasonDbId = existingStudy.getSeasons().get(0); // It is assumed that the study has only one season + if(StringUtils.isNotBlank(seasonDbId)) { + String seasonYear = seasonDbIdToYear(seasonDbId, program.getId()); + existingStudy.setSeasons(Collections.singletonList(seasonYear)); + } + pendingStudy = new PendingImportObject<>( + ImportObjectState.EXISTING, + (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), + UUID.fromString(xref.getReferenceId()) + ); + studyMap.put( + Utilities.removeProgramKeyAndUnknownAdditionalData(getterFunction.apply(existingStudy), program.getKey()), + pendingStudy + ); + return pendingStudy; + } + + // TODO: used by both workflows + private String seasonDbIdToYear(String seasonDbId, UUID programId) { + String year = null; + // TODO: add season objects to redis cache then just extract year from those + // removing this for now here + //if (this.seasonDbIdToYearCache.containsKey(seasonDbId)) { // get it from cache if possible + // year = this.seasonDbIdToYearCache.get(seasonDbId); + //} else { + year = seasonDbIdToYearFromDatabase(seasonDbId, programId); + // this.seasonDbIdToYearCache.put(seasonDbId, year); + //} + return year; + } + + // TODO: used by both workflows + private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { + BrAPISeason season = null; + try { + season = this.brAPISeasonDAO.getSeasonById(seasonDbId, programId); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + } + Integer yearInt = (season == null) ? null : season.getYear(); + return (yearInt == null) ? "" : yearInt.toString(); + } + + // TODO: used by both worflows + /** + * Fetches a list of BrAPI studies by their study database IDs for a given program. + * + * This method queries the BrAPIStudyDAO to retrieve studies based on the provided study database IDs and the program. + * It ensures that all requested study database IDs are found in the result set, throwing an IllegalStateException if any are missing. + * + * @param studyDbIds a Set of Strings representing the study database IDs to fetch + * @param program the Program object representing the program context in which to fetch studies + * @return a List of BrAPIStudy objects matching the provided study database IDs + * + * @throws ApiException if there is an issue fetching the studies + * @throws IllegalStateException if any requested study database IDs are not found in the result set + */ + public List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { + List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); + if (studies.size() != studyDbIds.size()) { + List missingIds = new ArrayList<>(studyDbIds); + missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); + throw new IllegalStateException( + "Study not found for studyDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + return studies; + } + +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java new file mode 100644 index 000000000..3ed7ca7a6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java @@ -0,0 +1,180 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +@Singleton +@Slf4j +public class SharedTrialService { + private final BrAPITrialDAO brAPITrialDAO; + + private final SharedStudyService studyService; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public SharedTrialService(BrAPITrialDAO brAPITrialDAO, + SharedStudyService studyService) { + this.brAPITrialDAO = brAPITrialDAO; + this.studyService = studyService; + } + + // TODO: also used in other workflow + /** + * Initializes trials for existing observation units. + * + * @param program The program object. + * @param observationUnitByNameNoScope A map containing observation units by name (without scope). + * @param trialByName A map containing trials by name. (will be modified in place) + * + */ + public void initializeTrialsForExistingObservationUnits(Program program, + Map> observationUnitByNameNoScope, + Map> trialByName) { + if(observationUnitByNameNoScope.size() > 0) { + Set trialDbIds = new HashSet<>(); + Set studyDbIds = new HashSet<>(); + + observationUnitByNameNoScope.values() + .forEach(pio -> { + BrAPIObservationUnit existingOu = pio.getBrAPIObject(); + if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { + throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); + } + + if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { + trialDbIds.add(existingOu.getTrialDbId()); + } else { + studyDbIds.add(existingOu.getStudyDbId()); + } + }); + + //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId + if(!studyDbIds.isEmpty()) { + try { + trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, program)); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + try { + List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); + if (trials.size() != trialDbIds.size()) { + List missingIds = new ArrayList<>(trialDbIds); + missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); + throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + + trials.forEach(trial -> processAndCacheTrial(trial, program, trialByName)); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + } + + /** + * Fetches trial DbIds for the given study DbIds by using the BrAPI studies API. + * + * @param studyDbIds The set of study DbIds for which to fetch trial DbIds. + * @param program The program associated with the studies. + * @return A set of trial DbIds corresponding to the provided study DbIds. + * @throws ApiException If there was an error while fetching the studies or if a study does not have a trial DbId. + * @throws IllegalStateException If the trial DbId is not set for an existing study. + */ + private Set fetchTrialDbidsForStudies(Set studyDbIds, Program program) throws ApiException { + Set trialDbIds = new HashSet<>(); + List studies = studyService.fetchStudiesByDbId(studyDbIds, program); + studies.forEach(study -> { + if (StringUtils.isBlank(study.getTrialDbId())) { + throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); + } + trialDbIds.add(study.getTrialDbId()); + }); + + return trialDbIds; + } + + /** + * This method processes an existing trial, retrieves the experiment ID from the trial's external references, + * and caches the trial with the corresponding experiment ID in a map. + * + * @param existingTrial The existing BrAPITrial object to be processed and cached. + * @param program The Program object associated with the trial. + * @param trialByNameNoScope The map to cache the trial by its name without program scope. (will be modified in place) + * + * @throws InternalServerException + */ + private void processAndCacheTrial( + BrAPITrial existingTrial, + Program program, + Map> trialByNameNoScope) { + + //get TrialId from existingTrial + BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) + .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); + UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); + + trialByNameNoScope.put( + Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), + new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); + } + + /** + * Initializes trials by name without scope for the given program. + * + * @param program the program to initialize trials for + * @param observationUnitByNameNoScope a map of observation units by name without scope + * @param experimentImportRows a list of experiment observation rows + * @return a map of trials by name with pending import objects + * + * @throws InternalServerException + */ + public Map> initializeTrialByNameNoScope(Program program, Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> trialByName = new HashMap<>(); + + initializeTrialsForExistingObservationUnits(program, observationUnitByNameNoScope, trialByName); + + List uniqueTrialNames = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID())) + .map(ExperimentObservation::getExpTitle) + .distinct() + .collect(Collectors.toList()); + try { + brAPITrialDAO.getTrialsByName(uniqueTrialNames, program).forEach(existingTrial -> + processAndCacheTrial(existingTrial, program, trialByName) + ); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + return trialByName; + } + +} \ No newline at end of file From 777b7e2a3edd53f91302344bca81dee5a353d2eb Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 20 May 2024 15:31:31 -0400 Subject: [PATCH 059/203] Added file headers and dynamic column parser --- .../services/pipeline/ProcessingStep.java | 1 - .../experiment/DynamicColumnParser.java | 64 +++++++++++++++++++ .../experiment/ExperimentUtilities.java | 18 ++++++ .../experiment/create/model/PendingData.java | 16 +++++ .../steps/GetExistingProcessingStep.java | 16 +++++ .../create/workflow/steps/ProcessStep.java | 23 +++++++ .../services/SharedStudyService.java | 16 +++++ .../services/SharedTrialService.java | 16 +++++ 8 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java index ca16792db..fb7f5e514 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java @@ -15,7 +15,6 @@ * limitations under the License. */ - package org.breedinginsight.brapps.importer.services.pipeline; public interface ProcessingStep { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java new file mode 100644 index 000000000..6b250facc --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java @@ -0,0 +1,64 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors.experiment; + +import lombok.Getter; +import tech.tablesaw.api.Table; +import tech.tablesaw.columns.Column; + +import java.util.ArrayList; +import java.util.List; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.TIMESTAMP_PREFIX; + +public class DynamicColumnParser { + + /** + * Parses dynamic columns from a table and separates them into phenotype and timestamp columns. + * + * @param data The table containing the dynamic columns. + * @param dynamicColumnNames An array of dynamic column names to be parsed. + * @return A DynamicColumnParseResult object containing the parsed phenotype and timestamp columns. + */ + public static DynamicColumnParseResult parse(Table data, String[] dynamicColumnNames) { + List> dynamicCols = data.columns(dynamicColumnNames); + List> phenotypeCols = new ArrayList<>(); + List> timestampCols = new ArrayList<>(); + + for (Column dynamicCol : dynamicCols) { + if (dynamicCol.name().startsWith(TIMESTAMP_PREFIX)) { + timestampCols.add(dynamicCol); + } else { + phenotypeCols.add(dynamicCol); + } + } + + return new DynamicColumnParseResult(phenotypeCols, timestampCols); + } + + @Getter + public static class DynamicColumnParseResult { + private final List> phenotypeCols; + private final List> timestampCols; + + public DynamicColumnParseResult(List> phenotypeCols, List> timestampCols) { + this.phenotypeCols = phenotypeCols; + this.timestampCols = timestampCols; + } + + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index ed685449d..f6cb96490 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -1,3 +1,19 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.breedinginsight.brapps.importer.services.processors.experiment; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; @@ -9,6 +25,8 @@ public class ExperimentUtilities { public static final CharSequence COMMA_DELIMITER = ","; + public static final String TIMESTAMP_PREFIX = "TS:"; + public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; public static List importRowsToExperimentObservations(List importRows) { return importRows.stream() diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java index 7d6fc74b9..213411078 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -1,3 +1,19 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; import lombok.*; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java index 10afb13a8..5bba6fd52 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java @@ -1,3 +1,19 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; import io.micronaut.context.annotation.Property; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java index 67127c103..8560f6f84 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java @@ -1,3 +1,19 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; @@ -6,9 +22,16 @@ public class ProcessStep implements ProcessingStep { + public ProcessStep() { + + } + @Override public ProcessedData process(PendingData input) { + + + // TODO: implement return new ProcessedData(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java index 80af3e2a2..0e9aef883 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java @@ -1,3 +1,19 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.breedinginsight.brapps.importer.services.processors.experiment.services; import io.micronaut.context.annotation.Property; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java index 3ed7ca7a6..d3d5efd1f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java @@ -1,3 +1,19 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.breedinginsight.brapps.importer.services.processors.experiment.services; import io.micronaut.context.annotation.Property; From 9ec7de200bdf0f106259eb056eada5babadc0a64 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 20 May 2024 20:33:18 -0400 Subject: [PATCH 060/203] create classes for processed data --- .../AppendOverwritePhenotypesWorkflow.java | 9 +- .../{HandleErr.java => Transaction.java} | 4 +- .../middleware/process/InitialData.java | 81 ++++++++ .../middleware/process/OverwrittenData.java | 177 ++++++++++++++++++ .../middleware/process/UnchangedData.java | 33 ++++ .../process/VisitedObservationData.java | 13 ++ .../process/brapi/PendingObservation.java | 138 ++++++-------- .../service/ObservationService.java | 10 +- .../service/ObservationVariableService.java | 10 +- 9 files changed, 369 insertions(+), 106 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{HandleErr.java => Transaction.java} (86%) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 2d8664968..96878ba62 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -3,9 +3,8 @@ import io.micronaut.context.annotation.Prototype; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.GetExistingBrAPIData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.HandleErr; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.Transaction; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ValidateAllRowsHaveIDs; -import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.Middleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; @@ -19,15 +18,15 @@ public class AppendOverwritePhenotypesWorkflow implements Workflow { ExpUnitMiddleware middleware; - Provider handleErrProvider; + Provider transactionProvider; Provider validateAllRowsHaveIDsProvider; Provider getExistingBrAPIDataProvider; @Inject - public AppendOverwritePhenotypesWorkflow(Provider handleErrProvider, + public AppendOverwritePhenotypesWorkflow(Provider transactionProvider, Provider validateAllRowsHaveIDsProvider, Provider getExistingBrAPIDataProvider) { - this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(handleErrProvider.get(), + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(transactionProvider.get(), validateAllRowsHaveIDsProvider.get(), getExistingBrAPIDataProvider.get()); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/HandleErr.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java similarity index 86% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/HandleErr.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java index 267643d63..5e3fd28e9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/HandleErr.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java @@ -5,7 +5,9 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; @Slf4j -public class HandleErr extends ExpUnitMiddleware { +public class Transaction extends ExpUnitMiddleware { + // TODO: add member for ExpUnitContext + @Override public boolean process(ExpUnitMiddlewareContext context) { return processNext(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java new file mode 100644 index 000000000..92f929a8b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java @@ -0,0 +1,81 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import com.google.gson.Gson; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapi.v2.services.BrAPIStudyService; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.BRAPI_REFERENCE_SOURCE; + +public class InitialData extends VisitedObservationData { + boolean isCommit; + String cellData; + String phenoColumnName; + ExperimentObservation row; + UUID trialId; + UUID studyId; + String unitId; + String studyYear; + BrAPIObservationUnit observationUnit; + User user; + Program program; + @Inject + StudyService studyService; + @Inject + Gson gson; + + public InitialData(boolean isCommit, + String cellData, + String phenoColumnName, + ExperimentObservation row, + UUID trialId, + UUID studyId, + String unitId, + String studyYear, + BrAPIObservationUnit observationUnit, User user, + Program program) { + this.isCommit = isCommit; + this.cellData = cellData; + this.phenoColumnName = phenoColumnName; + this.row = row; + this.trialId = trialId; + this.studyId = studyId; + this.unitId = unitId; + this.studyYear = studyYear; + this.observationUnit = observationUnit; + this.user = user; + this.program = program; + } + @Override + public Optional> getValidationErrors() { + return Optional.empty(); + } + + @Override + public PendingImportObject constructPendingObservation() { + String seasonDbId = studyService.seasonDbIdToYear(studyYear, program.getId()); + + // Generate a new ID for the observation + UUID observationID = UUID.randomUUID(); + + // Construct the new observation + BrAPIObservation newObservation = row.constructBrAPIObservation(cellData, phenoColumnName, seasonDbId, observationUnit, isCommit, program, user, BRAPI_REFERENCE_SOURCE, trialId, studyId, UUID.fromString(unitId), observationID); + + // Construct a pending observation with a status set to NEW + return new PendingImportObject<>(ImportObjectState.NEW, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); + + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java new file mode 100644 index 000000000..636870243 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java @@ -0,0 +1,177 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import io.micronaut.http.HttpStatus; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +public class OverwrittenData extends VisitedObservationData { + @Inject + Gson gson; + boolean canOverwrite; + boolean isCommit; + String unitId; + String phenoColumnName; + String timestampColumnName; + String cellData; + String timestamp; + String reason; + BrAPIObservation observation; + UUID userId; + Program program; + + public OverwrittenData(boolean canOverwrite, + boolean isCommit, + String unitId, + String phenoColumnName, + String timestampColumnName, + String cellData, + String timestamp, + String reason, + BrAPIObservation observation, + UUID userId, + Program program) { + this.canOverwrite = canOverwrite; + this.isCommit = isCommit; + this.unitId = unitId; + this.phenoColumnName = phenoColumnName; + this.timestampColumnName = timestampColumnName; + this.cellData = cellData; + this.timestamp = timestamp; + this.reason = reason; + this.observation = observation; + this.userId = userId; + this.program = program; + } + + @Override + public Optional> getValidationErrors() { + List errors = null; + + // Errors for trying to change protected data + if (!canOverwrite) { + errors = new ArrayList<>(); + if (!isValueMatched()) { + errors.add(new ValidationError(phenoColumnName, String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", unitId, phenoColumnName), HttpStatus.UNPROCESSABLE_ENTITY)); + } + if (!isTimestampMatched()) { + errors.add(new ValidationError(timestampColumnName, String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", unitId, timestampColumnName), HttpStatus.UNPROCESSABLE_ENTITY)); + } + } + + return Optional.ofNullable(errors); + } + + @Override + public PendingImportObject constructPendingObservation() { + // Construct a pending observation with a status set to MUTATED + PendingImportObject pendingUpdatedObservation = new PendingImportObject<>(ImportObjectState.MUTATED, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); + BrAPIObservation update = pendingUpdatedObservation.getBrAPIObject(); + String original = null; + + if (!isValueMatched()) { + // Update the observation value + update.setValue(cellData); + + // Record original observation value for changelog entry + original = observation.getValue(); + } + + if (!isTimestampMatched()) { + // Update the timestamp + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timestamp)); + update.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + + // Add original timestamp to changelog entry + original = Optional.ofNullable(original).map(o -> o + " " + observation.getObservationTimeStamp()).orElse(String.valueOf(observation.getObservationTimeStamp())); + } + + // If the change is to be committed, attach a record of the change as BrAPI observation additional info + if (isCommit) { + // Create the changelog field in observation additional info if it does not already exist + createAdditionalInfoChangeLog(update); + + // Construct a changelog entry + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String rightNow = formatter.format(OffsetDateTime.now()); + ChangeLogEntry entry = new ChangeLogEntry(original, Optional.ofNullable(reason).orElse(""), userId, rightNow); + + // Add the entry to the changelog + update.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(entry).getAsJsonObject()); + } + + return pendingUpdatedObservation; + } + + private void createAdditionalInfoChangeLog(BrAPIObservation update) { + if (update.getAdditionalInfo().isJsonNull()) { + update.setAdditionalInfo(new JsonObject()); + update.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + if (update.getAdditionalInfo() != null && !update.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { + update.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + } + + private boolean isValueMatched() { + return !cellData.equals(observation.getValue()); + } + + private boolean isTimestampMatched() { + if (timestamp == null) { + return observation.getObservationTimeStamp() == null; + } else { + return !OffsetDateTime.parse(timestamp).equals(observation.getObservationTimeStamp()); + } + } +// private void validateTimeStampValue(String value, +// String columnHeader, ValidationErrors validationErrors, int row) { +// if (StringUtils.isBlank(value)) { +// log.debug(String.format("skipping validation of observation timestamp because there is no value.\n\tvariable: %s\n\trow: %d", columnHeader, row)); +// return; +// } +// if (!validDateValue(value) && !validDateTimeValue(value)) { +// addRowError(columnHeader, "Incorrect datetime format detected. Expected YYYY-MM-DD or YYYY-MM-DDThh:mm:ss+hh:mm", validationErrors, row); +// } +// +// } + + private boolean validDateValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } + + private boolean validDateTimeValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java new file mode 100644 index 000000000..8c252b25c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java @@ -0,0 +1,33 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import java.util.List; +import java.util.Optional; + +public class UnchangedData extends VisitedObservationData { + BrAPIObservation observation; + Program program; + + public UnchangedData(BrAPIObservation observation, Program program) { + this.observation = observation; + this.program = program; + } + @Override + public Optional> getValidationErrors() { + return Optional.empty(); + } + + @Override + public PendingImportObject constructPendingObservation() { + // Construct a pending observation with a status set to EXISTING + PendingImportObject pendingExistingObservation = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); + + return pendingExistingObservation; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java new file mode 100644 index 000000000..b224fc57e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java @@ -0,0 +1,13 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; + +import java.util.List; +import java.util.Optional; + +public abstract class VisitedObservationData { + abstract public Optional> getValidationErrors(); + abstract public PendingImportObject constructPendingObservation(); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java index 6413e6c59..eb3161046 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java @@ -4,6 +4,8 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; import io.micronaut.http.HttpStatus; +import io.reactivex.rxjava3.core.Completable; +import io.reactivex.rxjava3.core.Maybe; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; @@ -26,6 +28,10 @@ import org.breedinginsight.brapps.importer.services.FileMappingUtil; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.InitialData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.OverwrittenData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.UnchangedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.VisitedObservationData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; @@ -186,7 +192,7 @@ public boolean process(ExpUnitMiddlewareContext context) { for (Column column : phenotypeCols) { String cellData = column.getString(rowNum); - // Generate hash for looking up prior observation + // Generate hash for looking up prior observation data String studyName = context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getStudyName(); String unitName = context.getExpUnitContext().getPendingObsUnitByOUId().get(unitId).getBrAPIObject().getObservationUnitName(); String phenoColumnName = column.name(); @@ -194,8 +200,10 @@ public boolean process(ExpUnitMiddlewareContext context) { // Get timestamp if associated column String timestamp = null; + String tsColumnName = null; if (tsColByPheno.containsKey(phenoColumnName)) { timestamp = tsColByPheno.get(phenoColumnName).getString(rowNum); + tsColumnName = tsColByPheno.get(phenoColumnName).name(); // If timestamp is not valid, set to midnight if (timestamp != null && !timestamp.isBlank() && (!observationService.validDateTimeValue(timestamp) || !observationService.validDateValue(timestamp))) { @@ -207,105 +215,67 @@ public boolean process(ExpUnitMiddlewareContext context) { } } - // Construct a pending observation if this is a prior observation + VisitedObservationData processedData = null; + // Is there prior observation data for this unit + var? if (observationByObsHash.containsKey(observationHash)) { // Clone the prior observation BrAPIObservation observation = gson.fromJson(gson.toJson(observationByObsHash.get(observationHash)), BrAPIObservation.class); - // Construct a pending observation - PendingImportObject pendingPriorObservation = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); - - // Are overwrites authorized? - boolean canOverwrite = context.getImportContext().isCommit() && "false".equals( row.getOverwrite() == null ? "false" : row.getOverwrite()); - String original = null; - - // Update the pending phenotypic data - if (!cellData.equals(observation.getValue())) { - pendingPriorObservation.getBrAPIObject().setValue(cellData); - pendingPriorObservation.setState(ImportObjectState.MUTATED); - - // Validation error if user has not chosen to overwrite existing data - if (StringUtils.isNotBlank(cellData) && !canOverwrite) { - ValidationError overwriteValErr = new ValidationError(tsColByPheno.get(phenoColumnName).name(), String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", unitId, phenoColumnName), HttpStatus.UNPROCESSABLE_ENTITY); - validationErrors.addError(rowNum, overwriteValErr); - } - - // Record original value in changelog entry - original = observation.getValue(); + // Is there a change to the prior data? + if (!cellData.equals(observation.getValue()) || (timestamp != null && !OffsetDateTime.parse(timestamp).equals(observation.getObservationTimeStamp()))) { + + // Is prior data protected? + boolean canOverwrite = context.getImportContext().isCommit() && "false".equals( row.getOverwrite() == null ? "false" : row.getOverwrite()); + + // create new instance of OverwrittenData + processedData = new OverwrittenData(canOverwrite, + context.getImportContext().isCommit(), + unitId, + phenoColumnName, + tsColumnName, + cellData, + timestamp, + Optional.ofNullable(row.getOverwriteReason()).orElse(""), + observation, + context.getImportContext().getUser().getId(), + program); + } else { + + // create new instance of UnchangedData + processedData = new UnchangedData(observation, program); } - // Update the pending timestamp - if (timestamp != null && !OffsetDateTime.parse(timestamp).equals(observation.getObservationTimeStamp())) { - DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; - String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timestamp)); - pendingPriorObservation.getBrAPIObject().setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); - pendingPriorObservation.setState(ImportObjectState.MUTATED); - - // Validation error if user has not chosen to overwrite existing timestamp - if (StringUtils.isNotBlank(cellData) && !canOverwrite) { - ValidationError overwriteValErr = new ValidationError(tsColByPheno.get(phenoColumnName).name(), String.format("Value already exists for ObsUnitId: %s, Timestamp: %s", unitId, tsColByPheno.get(phenoColumnName).name()), HttpStatus.UNPROCESSABLE_ENTITY); - validationErrors.addError(rowNum, overwriteValErr); - } - - // Add original timestamp to changelog entry - original = Optional.ofNullable(original).map(o -> o + " " + observation.getObservationTimeStamp()).orElse(String.valueOf(observation.getObservationTimeStamp())); - - } - - // Record any updates as BrAPI observation additional info - if (context.getImportContext().isCommit()) { - - // Create the changelog field in observation additional info if it does not already exist - if (pendingPriorObservation.getBrAPIObject().getAdditionalInfo().isJsonNull()) { - pendingPriorObservation.getBrAPIObject().setAdditionalInfo(new JsonObject()); - pendingPriorObservation.getBrAPIObject().getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); - } - if (pendingPriorObservation.getBrAPIObject().getAdditionalInfo() != null && !pendingPriorObservation.getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { - pendingPriorObservation.getBrAPIObject().getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); - } - - // Construct a changelog entry - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); - String rightNow = formatter.format(OffsetDateTime.now()); - String reason = Optional.ofNullable(row.getOverwriteReason()).orElse(""); - ChangeLogEntry entry = new ChangeLogEntry(original, reason, context.getImportContext().getUser().getId(), rightNow); - - // Add the entry to the changelog - pendingPriorObservation.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(entry).getAsJsonObject()); - } - - // Set the pending prior observation in the pending import for the row - mappedImportRow.getObservations().add(pendingPriorObservation); - - // Add pending observation to map - pendingObservationByHash.put(observationHash, pendingPriorObservation); - - // Construct a pending observation for new data if no prior observation } else { - UUID trialId = pendingTrial.getId(); - UUID studyId = context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getId(); - String studyYear = context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getSeasons().get(0); - String seasonDbId = studyService.seasonDbIdToYear(studyYear, program.getId()); // Clone the observation unit BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getExpUnitContext().getPendingObsUnitByOUId().get(row.getExpUnitId()).getBrAPIObject()), BrAPIObservationUnit.class); - // Generate a new ID for the observation - UUID observationID = UUID.randomUUID(); + // create new instance of InitialData + processedData = new InitialData(context.getImportContext().isCommit(), + cellData, + phenoColumnName, + row, + pendingTrial.getId(), + context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getId(), + unitId, + context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getSeasons().get(0), + observationUnit, + context.getImportContext().getUser(), + context.getImportContext().getProgram()); + } - // Construct the new observation - BrAPIObservation newObservation = row.constructBrAPIObservation(cellData, phenoColumnName, seasonDbId, observationUnit, context.getImportContext().isCommit(), program, context.getImportContext().getUser(), BRAPI_REFERENCE_SOURCE, trialId, studyId, UUID.fromString(unitId), observationID); + // Validate processed data + processedData.getValidationErrors().ifPresent(errList -> errList.forEach(e->validationErrors.addError(rowNum, e))); - // Construct a pending observation - PendingImportObject pendingNewObservation = new PendingImportObject<>(ImportObjectState.NEW, newObservation); + // Construct a pending observation + PendingImportObject pendingProcessedData = processedData.constructPendingObservation(); - // Set the new pending observation in the pending import for the row - mappedImportRow.getObservations().add(pendingNewObservation); + // Set the new pending observation in the pending import for the row + mappedImportRow.getObservations().add(pendingProcessedData); - // Add pending observation to map - pendingObservationByHash.put(observationHash, pendingNewObservation); - } + // Add pending observation to map + pendingObservationByHash.put(observationHash, pendingProcessedData); } // Set the pending import for the row diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index 67177dad5..38180d402 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -14,6 +14,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.OverwrittenData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; @@ -180,14 +181,7 @@ private void validateObservation(String importHash) { ); // create the changelog field in additional info if it does not already exist - if (pendingObservation.getAdditionalInfo().isJsonNull()) { - pendingObservation.setAdditionalInfo(new JsonObject()); - pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); - } - - if (pendingObservation.getAdditionalInfo() != null && !pendingObservation.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { - pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); - } + OverwrittenData.createChangeLog(pendingObservation); // add a new entry to the changelog pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java index bd4ced017..1671d77ee 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java @@ -14,6 +14,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.OverwrittenData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; @@ -170,14 +171,7 @@ private void validateObservation(String importHash) { ); // create the changelog field in additional info if it does not already exist - if (pendingObservation.getAdditionalInfo().isJsonNull()) { - pendingObservation.setAdditionalInfo(new JsonObject()); - pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); - } - - if (pendingObservation.getAdditionalInfo() != null && !pendingObservation.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { - pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); - } + OverwrittenData.createChangeLog(pendingObservation); // add a new entry to the changelog pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); From 92af4693b5742cb7b770687734f558f902f7e127 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 21 May 2024 11:15:17 -0400 Subject: [PATCH 061/203] create validator classes --- .../validate/field/DateValidator.java | 61 +++++++++++++++++ .../validate/field/FieldValidator.java | 28 ++++++++ .../validate/field/NominalValidator.java | 59 +++++++++++++++++ .../validate/field/NumericalValidator.java | 66 +++++++++++++++++++ .../validate/field/ObservationValidator.java | 12 ++++ .../validate/field/OrdinalValidator.java | 59 +++++++++++++++++ .../service/ObservationService.java | 31 +++++++++ 7 files changed, 316 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/FieldValidator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NominalValidator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NumericalValidator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/OrdinalValidator.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java new file mode 100644 index 000000000..2102566ec --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java @@ -0,0 +1,61 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; + +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.model.Trait; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; +import static org.breedinginsight.dao.db.enums.DataType.DATE; + +@Slf4j +@Singleton +public class DateValidator implements ObservationValidator { + @Inject + ObservationService observationService; + + public DateValidator(ObservationService observationService) { + this.observationService = observationService; + } + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + if (observationService.isBlankObservation(value)) { + log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + if (observationService.isNAObservation(value)) { + log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + // Is this a timestamp field? + if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + + } else { + + // skip if there is no trait data + if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { + return Optional.empty(); + } + + // skip if this is not a date trait + if (!DATE.equals(variable.getScale().getDataType())) { + return Optional.empty(); + } + + if (!observationService.validDateValue(value)) { + return Optional.of(new ValidationError(fieldName, "Incorrect date format detected. Expected YYYY-MM-DD", HttpStatus.UNPROCESSABLE_ENTITY)); + } + } + + + + return Optional.empty(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/FieldValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/FieldValidator.java new file mode 100644 index 000000000..df7ec338a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/FieldValidator.java @@ -0,0 +1,28 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; + +import io.micronaut.context.annotation.Primary; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.model.Trait; + +import javax.inject.Singleton; +import java.util.List; +import java.util.Optional; + +@Primary +@Singleton +public class FieldValidator implements ObservationValidator { + private final List validators; + + public FieldValidator(List validators) { + this.validators = validators; + } + + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + return validators.stream() + .map(validator->validator.validateField(fieldName, value, variable)) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NominalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NominalValidator.java new file mode 100644 index 000000000..38bb6c6a3 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NominalValidator.java @@ -0,0 +1,59 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; + +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.model.Trait; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; +import static org.breedinginsight.dao.db.enums.DataType.NOMINAL; + +@Slf4j +@Singleton +public class NominalValidator implements ObservationValidator { + @Inject + ObservationService observationService; + + public NominalValidator(ObservationService observationService) { + this.observationService = observationService; + } + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + if (observationService.isBlankObservation(value)) { + log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + if (observationService.isNAObservation(value)) { + log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + // Skip if field is a timestamp + if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + return Optional.empty(); + } + + // Skip if there is no trait data + if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { + return Optional.empty(); + } + + // Skip if this is not an ordinal trait + if (!NOMINAL.equals(variable.getScale().getDataType())) { + return Optional.empty(); + } + + // Validate categories + if (!observationService.validCategory(variable.getScale().getCategories(), value)) { + return Optional.of(new ValidationError(fieldName, "Undefined nominal category detected", HttpStatus.UNPROCESSABLE_ENTITY)); + } + + return Optional.empty(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NumericalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NumericalValidator.java new file mode 100644 index 000000000..cf7165818 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NumericalValidator.java @@ -0,0 +1,66 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; + +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.model.Trait; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.math.BigDecimal; +import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; +import static org.breedinginsight.dao.db.enums.DataType.NUMERICAL; + +@Slf4j +@Singleton +public class NumericalValidator implements ObservationValidator { + @Inject + ObservationService observationService; + + public NumericalValidator(ObservationService observationService) { + this.observationService = observationService; + } + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + if (observationService.isBlankObservation(value)) { + log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + if (observationService.isNAObservation(value)) { + log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + // Skip if field is a timestamp + if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + return Optional.empty(); + } + + // Skip if there is no trait data + if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { + return Optional.empty(); + } + + // Skip if this is not a numerical trait + if (!NUMERICAL.equals(variable.getScale().getDataType())) { + return Optional.empty(); + } + + Optional number = observationService.validNumericValue(value); + Optional validationError = number + .flatMap(num -> { + if (!observationService.validNumericRange(num, variable.getScale())) { + return Optional.of(new ValidationError(fieldName, "Value outside of min/max range detected", HttpStatus.UNPROCESSABLE_ENTITY)); + } + return Optional.empty(); + }) + .or(() -> Optional.of(new ValidationError(fieldName, "Non-numeric text detected", HttpStatus.UNPROCESSABLE_ENTITY))); + + return validationError; + + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java new file mode 100644 index 000000000..494fcc13c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java @@ -0,0 +1,12 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; + +import io.micronaut.core.order.Ordered; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.model.Trait; + +import java.util.Optional; + +@FunctionalInterface +public interface ObservationValidator extends Ordered { + Optional validateField(String fieldName, String value, Trait variable); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/OrdinalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/OrdinalValidator.java new file mode 100644 index 000000000..cd04c56f5 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/OrdinalValidator.java @@ -0,0 +1,59 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; + +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.model.Trait; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; +import static org.breedinginsight.dao.db.enums.DataType.ORDINAL; + +@Slf4j +@Singleton +public class OrdinalValidator implements ObservationValidator { + @Inject + ObservationService observationService; + + public OrdinalValidator(ObservationService observationService) { + this.observationService = observationService; + } + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + if (observationService.isBlankObservation(value)) { + log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + if (observationService.isNAObservation(value)) { + log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + // Skip if field is a timestamp + if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + return Optional.empty(); + } + + // Skip if there is no trait data + if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { + return Optional.empty(); + } + + // Skip if this is not an ordinal trait + if (!ORDINAL.equals(variable.getScale().getDataType())) { + return Optional.empty(); + } + + // Validate categories + if (!observationService.validCategory(variable.getScale().getCategories(), value)) { + return Optional.of(new ValidationError(fieldName, "Undefined ordinal category detected", HttpStatus.UNPROCESSABLE_ENTITY)); + } + + return Optional.empty(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index 38180d402..641aa153d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -3,12 +3,14 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -19,10 +21,12 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; +import org.breedinginsight.model.Scale; import org.breedinginsight.model.Trait; import org.breedinginsight.utilities.Utilities; import tech.tablesaw.columns.Column; +import java.math.BigDecimal; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; @@ -30,6 +34,33 @@ import java.util.stream.Collectors; public class ObservationService { + public boolean validCategory(List categories, String value) { + Set categoryValues = categories.stream() + .map(category -> category.getValue().toLowerCase()) + .collect(Collectors.toSet()); + return categoryValues.contains(value.toLowerCase()); + } + public boolean validNumericRange(BigDecimal value, Scale validValues) { + // account for empty min or max in valid determination + return (validValues.getValidValueMin() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMin())) >= 0) && + (validValues.getValidValueMax() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMax())) <= 0); + } + public Optional validNumericValue(String value) { + BigDecimal number; + try { + number = new BigDecimal(value); + } catch (NumberFormatException e) { + return Optional.empty(); + } + return Optional.of(number); + } + + public boolean isBlankObservation(String value) { + return StringUtils.isBlank(value); + } + public boolean isNAObservation(String value){ + return value.equalsIgnoreCase("NA"); + } public boolean validDateTimeValue(String value) { DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; try { From 99b17e2ccdd091a45e85bff99336d408ae8128b6 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 21 May 2024 16:39:43 -0400 Subject: [PATCH 062/203] use field validator --- .../middleware/process/InitialData.java | 16 ++++- .../middleware/process/OverwrittenData.java | 18 +++++- .../process/brapi/PendingObservation.java | 58 +++++++++++-------- .../validate/field/DateValidator.java | 14 +++-- .../validate/field/ObservationValidator.java | 3 +- 5 files changed, 74 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java index 92f929a8b..b048ed87d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java @@ -4,16 +4,18 @@ import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.model.v1.response.ValidationError; -import org.breedinginsight.brapi.v2.services.BrAPIStudyService; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; import org.breedinginsight.model.User; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -24,6 +26,7 @@ public class InitialData extends VisitedObservationData { boolean isCommit; String cellData; String phenoColumnName; + Trait trait; ExperimentObservation row; UUID trialId; UUID studyId; @@ -33,6 +36,8 @@ public class InitialData extends VisitedObservationData { User user; Program program; @Inject + FieldValidator fieldValidator; + @Inject StudyService studyService; @Inject Gson gson; @@ -40,6 +45,7 @@ public class InitialData extends VisitedObservationData { public InitialData(boolean isCommit, String cellData, String phenoColumnName, + Trait trait, ExperimentObservation row, UUID trialId, UUID studyId, @@ -50,6 +56,7 @@ public InitialData(boolean isCommit, this.isCommit = isCommit; this.cellData = cellData; this.phenoColumnName = phenoColumnName; + this.trait = trait; this.row = row; this.trialId = trialId; this.studyId = studyId; @@ -61,7 +68,12 @@ public InitialData(boolean isCommit, } @Override public Optional> getValidationErrors() { - return Optional.empty(); + List errors = new ArrayList<>(); + + // Validate observation value + fieldValidator.validateField(phenoColumnName, cellData, trait).ifPresent(errors::add); + + return Optional.ofNullable(errors.isEmpty() ? null : errors); } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java index 636870243..40e3f72f5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java @@ -10,7 +10,9 @@ import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field.FieldValidator; import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; @@ -23,11 +25,14 @@ import java.util.UUID; public class OverwrittenData extends VisitedObservationData { + @Inject + FieldValidator fieldValidator; @Inject Gson gson; boolean canOverwrite; boolean isCommit; String unitId; + Trait trait; String phenoColumnName; String timestampColumnName; String cellData; @@ -40,6 +45,7 @@ public class OverwrittenData extends VisitedObservationData { public OverwrittenData(boolean canOverwrite, boolean isCommit, String unitId, + Trait trait, String phenoColumnName, String timestampColumnName, String cellData, @@ -51,6 +57,7 @@ public OverwrittenData(boolean canOverwrite, this.canOverwrite = canOverwrite; this.isCommit = isCommit; this.unitId = unitId; + this.trait = trait; this.phenoColumnName = phenoColumnName; this.timestampColumnName = timestampColumnName; this.cellData = cellData; @@ -63,11 +70,10 @@ public OverwrittenData(boolean canOverwrite, @Override public Optional> getValidationErrors() { - List errors = null; + List errors = new ArrayList<>(); // Errors for trying to change protected data if (!canOverwrite) { - errors = new ArrayList<>(); if (!isValueMatched()) { errors.add(new ValidationError(phenoColumnName, String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", unitId, phenoColumnName), HttpStatus.UNPROCESSABLE_ENTITY)); } @@ -76,7 +82,13 @@ public Optional> getValidationErrors() { } } - return Optional.ofNullable(errors); + // Validate observation value + fieldValidator.validateField(phenoColumnName, cellData, trait).ifPresent(errors::add); + + // Validate timestamp + fieldValidator.validateField(timestampColumnName, timestamp, null).ifPresent(errors::add); + + return Optional.ofNullable(errors.isEmpty() ? null : errors); } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java index eb3161046..bba8f886c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java @@ -1,12 +1,8 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi; import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import io.micronaut.http.HttpStatus; -import io.reactivex.rxjava3.core.Completable; -import io.reactivex.rxjava3.core.Maybe; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; @@ -20,7 +16,6 @@ import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; -import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -32,7 +27,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.OverwrittenData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.UnchangedData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.VisitedObservationData; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; @@ -44,13 +39,11 @@ import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; -import org.jooq.impl.QOM; import tech.tablesaw.api.Table; import tech.tablesaw.columns.Column; import javax.inject.Inject; import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; import java.util.*; import java.util.stream.Collectors; @@ -65,6 +58,7 @@ public class PendingObservation extends ExpUnitMiddleware { BrAPIObservationDAO brAPIObservationDAO; FileMappingUtil fileMappingUtil; Gson gson; + FieldValidator fieldValidator; @Inject public PendingObservation(StudyService studyService, @@ -72,7 +66,8 @@ public PendingObservation(StudyService studyService, BrAPIObservationDAO brAPIObservationDAO, ObservationService observationService, FileMappingUtil fileMappingUtil, - Gson gson) { + Gson gson, + FieldValidator fieldValidator) { this.studyService = studyService; this.observationVariableService = observationVariableService; this.brAPIObservationDAO = brAPIObservationDAO; @@ -126,6 +121,16 @@ public boolean process(ExpUnitMiddlewareContext context) { Program program = context.getImportContext().getProgram(); List traits = observationVariableService.fetchTraitsByName(varNames, program); + // Map trait by phenotype column name + Map traitByPhenoColName = traits.stream().collect( + Collectors.toMap( + trait -> trait.getObservationVariableName().toUpperCase(), // Use uppercase keys for case-insensitivity + trait -> trait, + (trait1, trait2) -> trait1, // Merge function + CaseInsensitiveMap::new // Supplier for creating a CaseInsensitiveMap + ) + ); + // Sort the traits to match the order of the headers in the import file List sortedTraits = fileMappingUtil.sortByField(List.copyOf(varNames), new ArrayList<>(traits), TraitEntity::getObservationVariableName); @@ -173,8 +178,7 @@ public boolean process(ExpUnitMiddlewareContext context) { // Build new pending observation data for each phenotype Map> pendingObservationByHash = new HashMap<>(); - - // Checking all import rows for data + // Process observation data for each row for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { Integer rowNum = i; ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); @@ -199,20 +203,20 @@ public boolean process(ExpUnitMiddlewareContext context) { String observationHash = observationService.getObservationHash(unitName, phenoColumnName, studyName); // Get timestamp if associated column - String timestamp = null; + var cell = new Object() { // mutable reference object to make timestamp accessible in anonymous methods + String timestamp = null; + }; String tsColumnName = null; if (tsColByPheno.containsKey(phenoColumnName)) { - timestamp = tsColByPheno.get(phenoColumnName).getString(rowNum); + cell.timestamp = tsColByPheno.get(phenoColumnName).getString(rowNum); tsColumnName = tsColByPheno.get(phenoColumnName).name(); // If timestamp is not valid, set to midnight - if (timestamp != null && !timestamp.isBlank() && (!observationService.validDateTimeValue(timestamp) || !observationService.validDateValue(timestamp))) { - timestamp += MIDNIGHT; + fieldValidator.validateField(tsColumnName, cell.timestamp, null).ifPresent(err->{ + cell.timestamp += MIDNIGHT; + validationErrors.addError(rowNum, err); + }); - // Add a validation error - ValidationError timestampValErr = new ValidationError(tsColByPheno.get(phenoColumnName).name(), String.format("Timestamp format is not valid for %s", tsColByPheno.get(phenoColumnName).name()), HttpStatus.UNPROCESSABLE_ENTITY); - validationErrors.addError(rowNum, timestampValErr); - } } VisitedObservationData processedData = null; @@ -223,19 +227,23 @@ public boolean process(ExpUnitMiddlewareContext context) { BrAPIObservation observation = gson.fromJson(gson.toJson(observationByObsHash.get(observationHash)), BrAPIObservation.class); // Is there a change to the prior data? - if (!cellData.equals(observation.getValue()) || (timestamp != null && !OffsetDateTime.parse(timestamp).equals(observation.getObservationTimeStamp()))) { + if (!cellData.equals(observation.getValue()) || (cell.timestamp != null && !OffsetDateTime.parse(cell.timestamp).equals(observation.getObservationTimeStamp()))) { // Is prior data protected? boolean canOverwrite = context.getImportContext().isCommit() && "false".equals( row.getOverwrite() == null ? "false" : row.getOverwrite()); - // create new instance of OverwrittenData + // Clone the trait + Trait changeTrait = gson.fromJson(gson.toJson(traitByPhenoColName.get(phenoColumnName)), Trait.class); + + // Create new instance of OverwrittenData processedData = new OverwrittenData(canOverwrite, context.getImportContext().isCommit(), unitId, + changeTrait, phenoColumnName, tsColumnName, cellData, - timestamp, + cell.timestamp, Optional.ofNullable(row.getOverwriteReason()).orElse(""), observation, context.getImportContext().getUser().getId(), @@ -248,13 +256,15 @@ public boolean process(ExpUnitMiddlewareContext context) { } else { - // Clone the observation unit + // Clone the observation unit and trait BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getExpUnitContext().getPendingObsUnitByOUId().get(row.getExpUnitId()).getBrAPIObject()), BrAPIObservationUnit.class); + Trait initialTrait = gson.fromJson(gson.toJson(traitByPhenoColName.get(phenoColumnName)), Trait.class); // create new instance of InitialData processedData = new InitialData(context.getImportContext().isCommit(), cellData, phenoColumnName, + initialTrait, row, pendingTrial.getId(), context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getId(), diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java index 2102566ec..d10c7c865 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java @@ -18,6 +18,8 @@ public class DateValidator implements ObservationValidator { @Inject ObservationService observationService; + private final String dateMessage = "Incorrect date format detected. Expected YYYY-MM-DD"; + private final String dateTimeMessage = "Incorrect datetime format detected. Expected YYYY-MM-DD or YYYY-MM-DDThh:mm:ss+hh:mm"; public DateValidator(ObservationService observationService) { this.observationService = observationService; @@ -36,26 +38,28 @@ public Optional validateField(String fieldName, String value, T // Is this a timestamp field? if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + if (!observationService.validDateValue(value) || !observationService.validDateTimeValue(value)) { + return Optional.of(new ValidationError(fieldName, dateTimeMessage, HttpStatus.UNPROCESSABLE_ENTITY)); + } } else { - // skip if there is no trait data + // Skip if there is no trait data if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { return Optional.empty(); } - // skip if this is not a date trait + // Skip if this is not a date trait if (!DATE.equals(variable.getScale().getDataType())) { return Optional.empty(); } + // Validate date if (!observationService.validDateValue(value)) { - return Optional.of(new ValidationError(fieldName, "Incorrect date format detected. Expected YYYY-MM-DD", HttpStatus.UNPROCESSABLE_ENTITY)); + return Optional.of(new ValidationError(fieldName, dateMessage, HttpStatus.UNPROCESSABLE_ENTITY)); } } - - return Optional.empty(); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java index 494fcc13c..825913732 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; +import io.micronaut.core.annotation.NonNull; import io.micronaut.core.order.Ordered; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.model.Trait; @@ -8,5 +9,5 @@ @FunctionalInterface public interface ObservationValidator extends Ordered { - Optional validateField(String fieldName, String value, Trait variable); + Optional validateField(@NonNull String fieldName, @NonNull String value, Trait variable); } From 9141e9b960cdbb269bbc179e6b255a5e52e5abb7 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 21 May 2024 20:48:13 -0400 Subject: [PATCH 063/203] create preview statistics class --- .../middleware/process/AppendStatistic.java | 87 +++++++++++++++++++ .../middleware/process/InitialData.java | 7 +- .../middleware/process/OverwrittenData.java | 7 +- .../middleware/process/UnchangedData.java | 5 ++ .../process/VisitedObservationData.java | 1 + .../process/brapi/PendingObservation.java | 20 +++-- .../validate/{field => }/DateValidator.java | 2 +- .../validate/{field => }/FieldValidator.java | 2 +- .../{field => }/NominalValidator.java | 2 +- .../{field => }/NumericalValidator.java | 2 +- .../{field => }/ObservationValidator.java | 2 +- .../{field => }/OrdinalValidator.java | 2 +- 12 files changed, 125 insertions(+), 14 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/{field => }/DateValidator.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/{field => }/FieldValidator.java (94%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/{field => }/NominalValidator.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/{field => }/NumericalValidator.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/{field => }/ObservationValidator.java (89%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/{field => }/OrdinalValidator.java (97%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java new file mode 100644 index 000000000..aeb88cfe8 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java @@ -0,0 +1,87 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; + +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; + +@Prototype +public class AppendStatistic { + private final HashSet environmentNames; + private final HashSet observationUnitIds; + private final HashSet gids; + private int newCount; + private int existingCount; + private int mutatedCount; + + public AppendStatistic() { + this.environmentNames = new HashSet<>(); + this.observationUnitIds = new HashSet<>(); + this.gids = new HashSet<>(); + this.newCount = 0; + this.existingCount = 0; + this.mutatedCount = 0; + } + + public int incrementNewCount(Integer value) { + int increment = 0; + if (value == null) { + increment = 1; + } else if (value >= 0) { + increment = value; + } + this.newCount += increment; + + return this.newCount; + } + public int incrementExistingCount(Integer value) { + int increment = 0; + if (value == null) { + increment = 1; + } else if (value >= 0) { + increment = value; + } + this.existingCount += increment; + + return this.existingCount; + } + public int incrementMutatedCount(Integer value) { + int increment = 0; + if (value == null) { + increment = 1; + } else if (value >= 0) { + increment = value; + } + this.mutatedCount += increment; + + return this.mutatedCount; + } + public void addEnvironmentName(String name) { + Optional.ofNullable(name).ifPresent(environmentNames::add); + } + public void addObservationUnitId(String id) { + Optional.ofNullable(id).ifPresent(observationUnitIds::add); + } + public void addGid(String gid) { + Optional.ofNullable(gid).ifPresent(gids::add); + } + public Map constructPreviewMap() { + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder().newObjectCount(environmentNames.size()).build(); + ImportPreviewStatistics observationUnitsStats = ImportPreviewStatistics.builder().newObjectCount(observationUnitIds.size()).build(); + ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder().newObjectCount(gids.size()).build(); + ImportPreviewStatistics newStats = ImportPreviewStatistics.builder().newObjectCount(newCount).build(); + ImportPreviewStatistics existingStats = ImportPreviewStatistics.builder().newObjectCount(existingCount).build(); + ImportPreviewStatistics mutatedStats = ImportPreviewStatistics.builder().newObjectCount(mutatedCount).build(); + + return Map.of( + "Environments", environmentStats, + "Observation_Units", observationUnitsStats, + "GIDs", gidStats, + "Observations", newStats, + "Existing_Observations", existingStats, + "Mutated_Observations", mutatedStats + ); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java index b048ed87d..48dd9924f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java @@ -7,7 +7,7 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; @@ -90,4 +90,9 @@ public PendingImportObject constructPendingObservation() { return new PendingImportObject<>(ImportObjectState.NEW, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); } + + @Override + public void updateTally(AppendStatistic statistic) { + statistic.incrementNewCount(1); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java index 40e3f72f5..a6d7c23ee 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java @@ -10,7 +10,7 @@ import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.FieldValidator; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; import org.breedinginsight.utilities.Utilities; @@ -133,6 +133,11 @@ public PendingImportObject constructPendingObservation() { return pendingUpdatedObservation; } + @Override + public void updateTally(AppendStatistic statistic) { + statistic.incrementMutatedCount(1); + } + private void createAdditionalInfoChangeLog(BrAPIObservation update) { if (update.getAdditionalInfo().isJsonNull()) { update.setAdditionalInfo(new JsonObject()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java index 8c252b25c..6c3245966 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java @@ -30,4 +30,9 @@ public PendingImportObject constructPendingObservation() { return pendingExistingObservation; } + + @Override + public void updateTally(AppendStatistic statistic) { + statistic.incrementExistingCount(1); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java index b224fc57e..bdd064b93 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java @@ -10,4 +10,5 @@ public abstract class VisitedObservationData { abstract public Optional> getValidationErrors(); abstract public PendingImportObject constructPendingObservation(); + abstract public void updateTally(AppendStatistic statistic); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java index bba8f886c..af82901a1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java @@ -23,11 +23,8 @@ import org.breedinginsight.brapps.importer.services.FileMappingUtil; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.InitialData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.OverwrittenData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.UnchangedData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.VisitedObservationData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.*; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; @@ -59,6 +56,7 @@ public class PendingObservation extends ExpUnitMiddleware { FileMappingUtil fileMappingUtil; Gson gson; FieldValidator fieldValidator; + AppendStatistic statistic; @Inject public PendingObservation(StudyService studyService, @@ -67,13 +65,15 @@ public PendingObservation(StudyService studyService, ObservationService observationService, FileMappingUtil fileMappingUtil, Gson gson, - FieldValidator fieldValidator) { + FieldValidator fieldValidator, + AppendStatistic statistic) { this.studyService = studyService; this.observationVariableService = observationVariableService; this.brAPIObservationDAO = brAPIObservationDAO; this.observationService = observationService; this.fileMappingUtil = fileMappingUtil; this.gson = gson; + this.statistic = statistic; } @Override @@ -278,6 +278,14 @@ public boolean process(ExpUnitMiddlewareContext context) { // Validate processed data processedData.getValidationErrors().ifPresent(errList -> errList.forEach(e->validationErrors.addError(rowNum, e))); + // Update import preview statistics + processedData.updateTally(statistic); + statistic.addEnvironmentName(studyName); + // TODO: change null values to actual data + // TODO: change signature to take two args, studyName and unitName + statistic.addObservationUnitId(null); + statistic.addGid(null); + // Construct a pending observation PendingImportObject pendingProcessedData = processedData.constructPendingObservation(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/DateValidator.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/DateValidator.java index d10c7c865..615da28ae 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/DateValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/FieldValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/FieldValidator.java similarity index 94% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/FieldValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/FieldValidator.java index df7ec338a..724aa01d3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/FieldValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/FieldValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; import io.micronaut.context.annotation.Primary; import org.breedinginsight.api.model.v1.response.ValidationError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NominalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NominalValidator.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NominalValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NominalValidator.java index 38bb6c6a3..7f2cf50df 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NominalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NominalValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NumericalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NumericalValidator.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NumericalValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NumericalValidator.java index cf7165818..d90d8d115 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NumericalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NumericalValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/ObservationValidator.java similarity index 89% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/ObservationValidator.java index 825913732..e378cd228 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/ObservationValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; import io.micronaut.core.annotation.NonNull; import io.micronaut.core.order.Ordered; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/OrdinalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/OrdinalValidator.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/OrdinalValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/OrdinalValidator.java index cd04c56f5..76fbcab04 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/OrdinalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/OrdinalValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; From df63e6db38d6447b59fe5e1e0c7c585767ae6426 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 21 May 2024 21:32:09 -0400 Subject: [PATCH 064/203] update --- .../middleware/process/brapi/PendingObservation.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java index af82901a1..e1d2361ce 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java @@ -178,7 +178,7 @@ public boolean process(ExpUnitMiddlewareContext context) { // Build new pending observation data for each phenotype Map> pendingObservationByHash = new HashMap<>(); - // Process observation data for each row + // Build pending import data maps for each row for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { Integer rowNum = i; ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); @@ -192,7 +192,7 @@ public boolean process(ExpUnitMiddlewareContext context) { mappedImportRow.setObservationUnit(context.getExpUnitContext().getPendingObsUnitByOUId().get(unitId)); mappedImportRow.setGermplasm(context.getExpUnitContext().getPendingGermplasmByOUId().get(unitId)); - // For each phenotype, construct the pending observations + // Assemble the pending observation data for all phenotypes for (Column column : phenotypeCols) { String cellData = column.getString(rowNum); From 2328d134110bb6e9a67039fcf5c7d438ac4503d8 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 21 May 2024 22:31:22 -0400 Subject: [PATCH 065/203] cleanup --- .../steps/GetExistingProcessingStep.java | 8 + .../experiment/model/ProcessedData.java | 1 - .../experiment/service/DatasetService.java | 334 +++++------ .../experiment/service/GermplasmService.java | 114 ++-- .../experiment/service/LocationService.java | 160 ++--- .../service/ObservationService.java | 446 +++++++------- .../service/ObservationUnitService.java | 344 +++++------ .../service/ObservationVariableService.java | 360 ++++++------ .../experiment/service/StatisticsService.java | 140 ++--- .../experiment/service/StudyService.java | 346 +++++------ .../experiment/service/TrialService.java | 165 +++--- .../experiment/service/ValidateService.java | 550 +++++++++--------- .../workflow/ExperimentWorkflowFactory.java | 2 +- 13 files changed, 1484 insertions(+), 1486 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java index 5cd943645..3ff8494ba 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -96,6 +96,10 @@ public PendingData process(ImportContext input) { return existing; } + private Map> initializeTrialByNameNoScope(Program program, Map> observationUnitByNameNoScope, List experimentImportRows) { + return null; + } + /** * Initializes the observation units for the given program and experimentImportRows. * @@ -282,6 +286,10 @@ private void initializeStudiesForExistingObservationUnits( } } + private List fetchStudiesByDbId(Set studyDbIds, Program program) { + return null; + } + /** * Initializes unique location names for a program. * diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java index 49fe30b12..9afbf189b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java @@ -6,7 +6,6 @@ import java.util.Map; @Data -@Builder @ToString @NoArgsConstructor public class ProcessedData { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index 399f23468..9810f8c62 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -100,70 +100,70 @@ public PendingImportObject constructPIOFromDataset(BrAPIListDe } // TODO: used by expunit worflow - public Map> initializeObsVarDatasetForExistingObservationUnits( - Map> trialByName, - Program program) { - Map> obsVarDatasetByName = new HashMap<>(); - - if (trialByName.size() > 0 && - trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { - String datasetId = trialByName.values().iterator().next().getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) - .getAsString(); - - try { - List existingDatasets = brAPIListDAO - .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, - program.getId(), - String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), - UUID.fromString(datasetId)); - if (existingDatasets == null || existingDatasets.isEmpty()) { - throw new InternalServerException("existing dataset summary not returned from brapi server"); - } - BrAPIListDetails dataSetDetails = brAPIListDAO - .getListById(existingDatasets.get(0).getListDbId(), program.getId()) - .getResult(); - processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); - } catch (ApiException e) { - log.error(Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - return obsVarDatasetByName; - } +// public Map> initializeObsVarDatasetForExistingObservationUnits( +// Map> trialByName, +// Program program) { +// Map> obsVarDatasetByName = new HashMap<>(); +// +// if (trialByName.size() > 0 && +// trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { +// String datasetId = trialByName.values().iterator().next().getBrAPIObject() +// .getAdditionalInfo() +// .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) +// .getAsString(); +// +// try { +// List existingDatasets = brAPIListDAO +// .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, +// program.getId(), +// String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), +// UUID.fromString(datasetId)); +// if (existingDatasets == null || existingDatasets.isEmpty()) { +// throw new InternalServerException("existing dataset summary not returned from brapi server"); +// } +// BrAPIListDetails dataSetDetails = brAPIListDAO +// .getListById(existingDatasets.get(0).getListDbId(), program.getId()) +// .getResult(); +// processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); +// } catch (ApiException e) { +// log.error(Utilities.generateApiExceptionLogMessage(e), e); +// throw new InternalServerException(e.toString(), e); +// } +// } +// return obsVarDatasetByName; +// } // TODO: used by create workflow - public Map> initializeObsVarDatasetByName(Program program, List experimentImportRows) { - Map> obsVarDatasetByName = new HashMap<>(); - - Optional> trialPIO = getTrialPIO(experimentImportRows); - - if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { - String datasetId = trialPIO.get().getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) - .getAsString(); - try { - List existingDatasets = brAPIListDAO - .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, - program.getId(), - String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), - UUID.fromString(datasetId)); - if (existingDatasets == null || existingDatasets.isEmpty()) { - throw new InternalServerException("existing dataset summary not returned from brapi server"); - } - BrAPIListDetails dataSetDetails = brAPIListDAO - .getListById(existingDatasets.get(0).getListDbId(), program.getId()) - .getResult(); - processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); - } catch (ApiException e) { - log.error(Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - return obsVarDatasetByName; - } +// public Map> initializeObsVarDatasetByName(Program program, List experimentImportRows) { +// Map> obsVarDatasetByName = new HashMap<>(); +// +// Optional> trialPIO = getTrialPIO(experimentImportRows); +// +// if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { +// String datasetId = trialPIO.get().getBrAPIObject() +// .getAdditionalInfo() +// .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) +// .getAsString(); +// try { +// List existingDatasets = brAPIListDAO +// .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, +// program.getId(), +// String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), +// UUID.fromString(datasetId)); +// if (existingDatasets == null || existingDatasets.isEmpty()) { +// throw new InternalServerException("existing dataset summary not returned from brapi server"); +// } +// BrAPIListDetails dataSetDetails = brAPIListDAO +// .getListById(existingDatasets.get(0).getListDbId(), program.getId()) +// .getResult(); +// processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); +// } catch (ApiException e) { +// log.error(Utilities.generateApiExceptionLogMessage(e), e); +// throw new InternalServerException(e.toString(), e); +// } +// } +// return obsVarDatasetByName; +// } // TODO: used by expunit workflow public Map> mapPendingObsDatasetByOUId( @@ -198,115 +198,115 @@ public void addObsVarsToDatasetDetails(PendingImportObject pio } // TODO: used by expunit workflow - public void fetchOrCreateDatasetPIO(ImportContext importContext, - PendingData pendingData, - ExpUnitContext expUnitContext, - List referencedTraits) throws UnprocessableEntityException { - PendingImportObject pio; - PendingImportObject trialPIO = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); - String name = String.format("Observation Dataset [%s-%s]", - program.getKey(), - trialPIO.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) - .getAsString()); - if (obsVarDatasetByName.containsKey(name)) { - pio = obsVarDatasetByName.get(name); - } else { - UUID id = UUID.randomUUID(); - BrAPIListDetails newDataset = importRow.constructDatasetDetails( - name, - id, - BRAPI_REFERENCE_SOURCE, - program, - trialPIO.getId().toString()); - pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); - trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); - if (ImportObjectState.EXISTING == trialPIO.getState()) { - trialPIO.setState(ImportObjectState.MUTATED); - } - obsVarDatasetByName.put(name, pio); - } - addObsVarsToDatasetDetails(pio, referencedTraits, program); - } +// public void fetchOrCreateDatasetPIO(ImportContext importContext, +// PendingData pendingData, +// ExpUnitContext expUnitContext, +// List referencedTraits) throws UnprocessableEntityException { +// PendingImportObject pio; +// PendingImportObject trialPIO = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); +// String name = String.format("Observation Dataset [%s-%s]", +// program.getKey(), +// trialPIO.getBrAPIObject() +// .getAdditionalInfo() +// .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) +// .getAsString()); +// if (obsVarDatasetByName.containsKey(name)) { +// pio = obsVarDatasetByName.get(name); +// } else { +// UUID id = UUID.randomUUID(); +// BrAPIListDetails newDataset = importRow.constructDatasetDetails( +// name, +// id, +// BRAPI_REFERENCE_SOURCE, +// program, +// trialPIO.getId().toString()); +// pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); +// trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); +// if (ImportObjectState.EXISTING == trialPIO.getState()) { +// trialPIO.setState(ImportObjectState.MUTATED); +// } +// obsVarDatasetByName.put(name, pio); +// } +// addObsVarsToDatasetDetails(pio, referencedTraits, program); +// } // TODO: used by create workflow - public void fetchOrCreateDatasetPIO(ImportContext importContext, - PendingData pendingData, - List referencedTraits) throws UnprocessableEntityException { - PendingImportObject pio; - PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); - - String name = String.format("Observation Dataset [%s-%s]", - program.getKey(), - trialPIO.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) - .getAsString()); - if (obsVarDatasetByName.containsKey(name)) { - pio = obsVarDatasetByName.get(name); - } else { - UUID id = UUID.randomUUID(); - BrAPIListDetails newDataset = importRow.constructDatasetDetails( - name, - id, - BRAPI_REFERENCE_SOURCE, - program, - trialPIO.getId().toString()); - pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); - trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); - if (ImportObjectState.EXISTING == trialPIO.getState()) { - trialPIO.setState(ImportObjectState.MUTATED); - } - obsVarDatasetByName.put(name, pio); - } - addObsVarsToDatasetDetails(pio, referencedTraits, program); - } +// public void fetchOrCreateDatasetPIO(ImportContext importContext, +// PendingData pendingData, +// List referencedTraits) throws UnprocessableEntityException { +// PendingImportObject pio; +// PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); +// +// String name = String.format("Observation Dataset [%s-%s]", +// program.getKey(), +// trialPIO.getBrAPIObject() +// .getAdditionalInfo() +// .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) +// .getAsString()); +// if (obsVarDatasetByName.containsKey(name)) { +// pio = obsVarDatasetByName.get(name); +// } else { +// UUID id = UUID.randomUUID(); +// BrAPIListDetails newDataset = importRow.constructDatasetDetails( +// name, +// id, +// BRAPI_REFERENCE_SOURCE, +// program, +// trialPIO.getId().toString()); +// pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); +// trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); +// if (ImportObjectState.EXISTING == trialPIO.getState()) { +// trialPIO.setState(ImportObjectState.MUTATED); +// } +// obsVarDatasetByName.put(name, pio); +// } +// addObsVarsToDatasetDetails(pio, referencedTraits, program); +// } // TODO: used by both workflows - public List commitNewPendingDatasetsToBrAPIStore(ImportContext importContext, PendingData pendingData) { - List newDatasetRequests = ProcessorData.getNewObjects(obsVarDatasetByName).stream().map(details -> { - BrAPIListNewRequest request = new BrAPIListNewRequest(); - request.setListName(details.getListName()); - request.setListType(details.getListType()); - request.setExternalReferences(details.getExternalReferences()); - request.setAdditionalInfo(details.getAdditionalInfo()); - request.data(details.getData()); - return request; - }).collect(Collectors.toList()); - List createdDatasets = new ArrayList<>(brAPIListDAO.createBrAPILists(newDatasetRequests, program.getId(), upload)); - for (BrAPIListSummary summary : createdDatasets) { - obsVarDatasetByName.get(summary.getListName()).getBrAPIObject().setListDbId(summary.getListDbId()); - } - return createdDatasets; - } +// public List commitNewPendingDatasetsToBrAPIStore(ImportContext importContext, PendingData pendingData) { +// List newDatasetRequests = ProcessorData.getNewObjects(obsVarDatasetByName).stream().map(details -> { +// BrAPIListNewRequest request = new BrAPIListNewRequest(); +// request.setListName(details.getListName()); +// request.setListType(details.getListType()); +// request.setExternalReferences(details.getExternalReferences()); +// request.setAdditionalInfo(details.getAdditionalInfo()); +// request.data(details.getData()); +// return request; +// }).collect(Collectors.toList()); +// List createdDatasets = new ArrayList<>(brAPIListDAO.createBrAPILists(newDatasetRequests, program.getId(), upload)); +// for (BrAPIListSummary summary : createdDatasets) { +// obsVarDatasetByName.get(summary.getListName()).getBrAPIObject().setListDbId(summary.getListDbId()); +// } +// return createdDatasets; +// } // TODO: used by both workflows - public List commitUpdatedPendingDatasetsToBrAPIStore(ImportContext importContext, PendingData pendingData) { - List updatedDatasets = new ArrayList<>(); - Map datasetNewDataById = ProcessorData - .getMutationsByObjectId(obsVarDatasetByName, BrAPIListSummary::getListDbId); - for (Map.Entry entry : datasetNewDataById.entrySet()) { - String id = entry.getKey(); - BrAPIListDetails dataset = entry.getValue(); - try { - List existingObsVarIds = brAPIListDAO.getListById(id, program.getId()).getResult().getData(); - List newObsVarIds = dataset - .getData() - .stream() - .filter(obsVarId -> !existingObsVarIds.contains(obsVarId)).collect(Collectors.toList()); - List obsVarIds = new ArrayList<>(existingObsVarIds); - obsVarIds.addAll(newObsVarIds); - dataset.setData(obsVarIds); - updatedDatasets.add(brAPIListDAO.updateBrAPIList(id, dataset, program.getId())); - } catch (ApiException e) { - log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException("Error saving experiment import", e); - } catch (Exception e) { - log.error("Error updating dataset observation variables: ", e); - throw new InternalServerException(e.getMessage(), e); - } - } - return updatedDatasets; - } +// public List commitUpdatedPendingDatasetsToBrAPIStore(ImportContext importContext, PendingData pendingData) { +// List updatedDatasets = new ArrayList<>(); +// Map datasetNewDataById = ProcessorData +// .getMutationsByObjectId(obsVarDatasetByName, BrAPIListSummary::getListDbId); +// for (Map.Entry entry : datasetNewDataById.entrySet()) { +// String id = entry.getKey(); +// BrAPIListDetails dataset = entry.getValue(); +// try { +// List existingObsVarIds = brAPIListDAO.getListById(id, program.getId()).getResult().getData(); +// List newObsVarIds = dataset +// .getData() +// .stream() +// .filter(obsVarId -> !existingObsVarIds.contains(obsVarId)).collect(Collectors.toList()); +// List obsVarIds = new ArrayList<>(existingObsVarIds); +// obsVarIds.addAll(newObsVarIds); +// dataset.setData(obsVarIds); +// updatedDatasets.add(brAPIListDAO.updateBrAPIList(id, dataset, program.getId())); +// } catch (ApiException e) { +// log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); +// throw new InternalServerException("Error saving experiment import", e); +// } catch (Exception e) { +// log.error("Error updating dataset observation variables: ", e); +// throw new InternalServerException(e.getMessage(), e); +// } +// } +// return updatedDatasets; +// } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java index b4e7127cd..88d085e41 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java @@ -78,65 +78,65 @@ public String getGIDFromGermplasmPIO(PendingImportObject pio) { } // TODO: used by expunit workflow - public Map> initializeGermplasmByGIDForExistingObservationUnits( - Map> unitByName, - Program program) { - Map> existingGermplasmByGID = new HashMap<>(); - - List existingGermplasms = new ArrayList<>(); - if(unitByName.size() > 0) { - Set germplasmDbIds = unitByName.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); - try { - existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); - } catch (ApiException e) { - log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - existingGermplasms.forEach(existingGermplasm -> { - BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) - .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); - existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); - }); - return existingGermplasmByGID; - } +// public Map> initializeGermplasmByGIDForExistingObservationUnits( +// Map> unitByName, +// Program program) { +// Map> existingGermplasmByGID = new HashMap<>(); +// +// List existingGermplasms = new ArrayList<>(); +// if(unitByName.size() > 0) { +// Set germplasmDbIds = unitByName.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); +// try { +// existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); +// } catch (ApiException e) { +// log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); +// throw new InternalServerException(e.toString(), e); +// } +// } +// +// existingGermplasms.forEach(existingGermplasm -> { +// BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) +// .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); +// existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); +// }); +// return existingGermplasmByGID; +// } // TODO: used by create worflow - public Map> initializeExistingGermplasmByGID(Program program, List experimentImportRows) { - Map> existingGermplasmByGID = new HashMap<>(); - - List existingGermplasms = new ArrayList<>(); - if(observationUnitByNameNoScope.size() > 0) { - Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); - try { - existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); - } catch (ApiException e) { - log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - List uniqueGermplasmGIDs = experimentImportRows.stream() - .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) - .map(ExperimentObservation::getGid) - .distinct() - .collect(Collectors.toList()); - - try { - existingGermplasms.addAll(this.getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); - } catch (ApiException e) { - log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - - existingGermplasms.forEach(existingGermplasm -> { - BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) - .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); - existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); - }); - return existingGermplasmByGID; - } +// public Map> initializeExistingGermplasmByGID(Program program, List experimentImportRows) { +// Map> existingGermplasmByGID = new HashMap<>(); +// +// List existingGermplasms = new ArrayList<>(); +// if(observationUnitByNameNoScope.size() > 0) { +// Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); +// try { +// existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); +// } catch (ApiException e) { +// log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); +// throw new InternalServerException(e.toString(), e); +// } +// } +// +// List uniqueGermplasmGIDs = experimentImportRows.stream() +// .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) +// .map(ExperimentObservation::getGid) +// .distinct() +// .collect(Collectors.toList()); +// +// try { +// existingGermplasms.addAll(this.getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); +// } catch (ApiException e) { +// log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); +// throw new InternalServerException(e.toString(), e); +// } +// +// existingGermplasms.forEach(existingGermplasm -> { +// BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) +// .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); +// existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); +// }); +// return existingGermplasmByGID; +// } // TODO: used by expunit workflow public Map> mapGermplasmByOUId( diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java index 2fe6815ca..faca12734 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java @@ -99,41 +99,41 @@ public Map> initializeLocationByNam } // TODO: used by create workflow - public Map> initializeUniqueLocationNames(Program program, List experimentImportRows) { - Map> locationByName = new HashMap<>(); - - List existingLocations = new ArrayList<>(); - if(studyByNameNoScope.size() > 0) { - Set locationDbIds = studyByNameNoScope.values() - .stream() - .map(study -> study.getBrAPIObject() - .getLocationDbId()) - .collect(Collectors.toSet()); - try { - existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); - } catch (ApiException e) { - log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - List uniqueLocationNames = experimentImportRows.stream() - .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) - .map(ExperimentObservation::getEnvLocation) - .distinct() - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - try { - existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); - } catch (ApiException e) { - log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - - existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); - return locationByName; - } +// public Map> initializeUniqueLocationNames(Program program, List experimentImportRows) { +// Map> locationByName = new HashMap<>(); +// +// List existingLocations = new ArrayList<>(); +// if(studyByNameNoScope.size() > 0) { +// Set locationDbIds = studyByNameNoScope.values() +// .stream() +// .map(study -> study.getBrAPIObject() +// .getLocationDbId()) +// .collect(Collectors.toSet()); +// try { +// existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); +// } catch (ApiException e) { +// log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); +// throw new InternalServerException(e.toString(), e); +// } +// } +// +// List uniqueLocationNames = experimentImportRows.stream() +// .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) +// .map(ExperimentObservation::getEnvLocation) +// .distinct() +// .filter(Objects::nonNull) +// .collect(Collectors.toList()); +// +// try { +// existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); +// } catch (ApiException e) { +// log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); +// throw new InternalServerException(e.toString(), e); +// } +// +// existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); +// return locationByName; +// } // TODO: used by expunit workflow public Map> mapPendingLocationByOUId( @@ -158,55 +158,55 @@ public Map> mapPendingLocationByOUI } // TODO: used by expunit workflow - private void fetchOrCreateLocationPIO(ImportContext importContext, ExpUnitContext expUnitContext) { - PendingImportObject pio; - String envLocationName = pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName(); - if (!locationByName.containsKey((importRow.getEnvLocation()))) { - ProgramLocation newLocation = new ProgramLocation(); - newLocation.setName(envLocationName); - pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); - this.locationByName.put(envLocationName, pio); - } - } +// private void fetchOrCreateLocationPIO(ImportContext importContext, ExpUnitContext expUnitContext) { +// PendingImportObject pio; +// String envLocationName = pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName(); +// if (!locationByName.containsKey((importRow.getEnvLocation()))) { +// ProgramLocation newLocation = new ProgramLocation(); +// newLocation.setName(envLocationName); +// pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); +// this.locationByName.put(envLocationName, pio); +// } +// } // TODO: used by create workflow - private void fetchOrCreateLocationPIO(ImportContext importContext) { - PendingImportObject pio; - String envLocationName = importRow.getEnvLocation(); - if (!locationByName.containsKey((importRow.getEnvLocation()))) { - ProgramLocation newLocation = new ProgramLocation(); - newLocation.setName(envLocationName); - pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); - this.locationByName.put(envLocationName, pio); - } - } +// private void fetchOrCreateLocationPIO(ImportContext importContext) { +// PendingImportObject pio; +// String envLocationName = importRow.getEnvLocation(); +// if (!locationByName.containsKey((importRow.getEnvLocation()))) { +// ProgramLocation newLocation = new ProgramLocation(); +// newLocation.setName(envLocationName); +// pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); +// this.locationByName.put(envLocationName, pio); +// } +// } // TODO: used by both workflows - public List commitNewPendingLocationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { - AuthenticatedUser actingUser = new AuthenticatedUser(upload.getUpdatedByUser().getName(), new ArrayList<>(), upload.getUpdatedByUser().getId(), new ArrayList<>()); - - List newLocations = ProcessorData.getNewObjects(this.locationByName) - .stream() - .map(location -> ProgramLocationRequest.builder() - .name(location.getName()) - .build()) - .collect(Collectors.toList()); - - List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); - // set the DbId to the for each newly created location - for (ProgramLocation createdLocation : createdLocations) { - String createdLocationName = createdLocation.getName(); - this.locationByName.get(createdLocationName) - .getBrAPIObject() - .setLocationDbId(createdLocation.getLocationDbId()); - } - return createdLocations; - } +// public List commitNewPendingLocationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { +// AuthenticatedUser actingUser = new AuthenticatedUser(upload.getUpdatedByUser().getName(), new ArrayList<>(), upload.getUpdatedByUser().getId(), new ArrayList<>()); +// +// List newLocations = ProcessorData.getNewObjects(this.locationByName) +// .stream() +// .map(location -> ProgramLocationRequest.builder() +// .name(location.getName()) +// .build()) +// .collect(Collectors.toList()); +// +// List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); +// // set the DbId to the for each newly created location +// for (ProgramLocation createdLocation : createdLocations) { +// String createdLocationName = createdLocation.getName(); +// this.locationByName.get(createdLocationName) +// .getBrAPIObject() +// .setLocationDbId(createdLocation.getLocationDbId()); +// } +// return createdLocations; +// } // TODO: used by both workflows - public List commitUpdatedPendingLocationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { - List updatedLocations = new ArrayList<>(); - - return updatedLocations; - } +// public List commitUpdatedPendingLocationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { +// List updatedLocations = new ArrayList<>(); +// +// return updatedLocations; +// } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index 641aa153d..4490cab30 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -86,238 +86,238 @@ public String getObservationHash(String observationUnitName, String variableName DigestUtils.sha256Hex(StringUtils.defaultString(studyName)); return DigestUtils.sha256Hex(concat); } - public Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { - Set ouDbIds = new HashSet<>(); - Set variableDbIds = new HashSet<>(); - Map variableNameByDbId = new HashMap<>(); - Map ouNameByDbId = new HashMap<>(); - Map studyNameByDbId = studyByNameNoScope.values() - .stream() - .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) - .map(PendingImportObject::getBrAPIObject) - .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); - - studyNameByDbId.keySet().forEach(studyDbId -> { - try { - brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { - if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { - ouDbIds.add(ou.getObservationUnitDbId()); - } - ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); - }); - } catch (ApiException e) { - throw new RuntimeException(e); - } - }); - - for (Trait referencedTrait : referencedTraits) { - variableDbIds.add(referencedTrait.getObservationVariableDbId()); - variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); - } - - List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); - - return existingObservations.stream() - .map(obs -> { - String studyName = studyNameByDbId.get(obs.getStudyDbId()); - String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); - String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); - - String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); - - return Map.entry(key, obs); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } +// public Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { +// Set ouDbIds = new HashSet<>(); +// Set variableDbIds = new HashSet<>(); +// Map variableNameByDbId = new HashMap<>(); +// Map ouNameByDbId = new HashMap<>(); +// Map studyNameByDbId = studyByNameNoScope.values() +// .stream() +// .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) +// .map(PendingImportObject::getBrAPIObject) +// .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); +// +// studyNameByDbId.keySet().forEach(studyDbId -> { +// try { +// brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { +// if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { +// ouDbIds.add(ou.getObservationUnitDbId()); +// } +// ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); +// }); +// } catch (ApiException e) { +// throw new RuntimeException(e); +// } +// }); +// +// for (Trait referencedTrait : referencedTraits) { +// variableDbIds.add(referencedTrait.getObservationVariableDbId()); +// variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); +// } +// +// List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); +// +// return existingObservations.stream() +// .map(obs -> { +// String studyName = studyNameByDbId.get(obs.getStudyDbId()); +// String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); +// String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); +// +// String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); +// +// return Map.entry(key, obs); +// }) +// .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); +// } // TODO: used with expUnit workflow - public void validateObservations(PendingData pendingData, - int rowNum, - ImportContext importContext, - ExpUnitContext expUnitContext, - List> phenotypeCols, - CaseInsensitiveMap colVarMap) { - for (Column phenoCol : phenotypeCols) { - String importHash; - String importObsValue = phenoCol.getString(rowNum); - - importHash = getImportObservationHash( - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), - getVariableNameFromColumn(phenoCol), - pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() - ); - - validateObservation(importHash); - } - } +// public void validateObservations(PendingData pendingData, +// int rowNum, +// ImportContext importContext, +// ExpUnitContext expUnitContext, +// List> phenotypeCols, +// CaseInsensitiveMap colVarMap) { +// for (Column phenoCol : phenotypeCols) { +// String importHash; +// String importObsValue = phenoCol.getString(rowNum); +// +// importHash = getImportObservationHash( +// pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), +// getVariableNameFromColumn(phenoCol), +// pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() +// ); +// +// validateObservation(importHash); +// } +// } // TODO: used with create workflow - public void validateObservations(PendingData pendingData, - int rowNum, - ImportContext importContext, - List> phenotypeCols, - CaseInsensitiveMap colVarMap) { - - - for (Column phenoCol : phenotypeCols) { - String importHash; - String importObsValue = phenoCol.getString(rowNum); - - importHash = getImportObservationHash(importRow, phenoCol.name()); - - validateObservation(importHash); - } - - } +// public void validateObservations(PendingData pendingData, +// int rowNum, +// ImportContext importContext, +// List> phenotypeCols, +// CaseInsensitiveMap colVarMap) { +// +// +// for (Column phenoCol : phenotypeCols) { +// String importHash; +// String importObsValue = phenoCol.getString(rowNum); +// +// importHash = getImportObservationHash(importRow, phenoCol.name()); +// +// validateObservation(importHash); +// } +// +// } // TODO: used by both workflows - private void validateObservation(String importHash) { - - - String importObsValue = phenoCol.getString(rowNum); - - - // error if import observation data already exists and user has not selected to overwrite - if (commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && - this.existingObsByObsHash.containsKey(importHash) && - StringUtils.isNotBlank(phenoCol.getString(rowNum)) && - !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { - addRowError( - phenoCol.name(), - String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), - validationErrors, rowNum - ); - - // preview case where observation has already been committed and the import row ObsVar data differs from what - // had been saved prior to import - } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { - - // add a change log entry when updating the value of an observation - if (commit) { - BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); - String timestamp = formatter.format(OffsetDateTime.now()); - String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; - String prior = ""; - if (isValueMatched(importHash, importObsValue)) { - prior.concat(existingObsByObsHash.get(importHash).getValue()); - } - if (timeStampColByPheno.containsKey(phenoCol.name()) && isTimestampMatched(importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { - prior = prior.isEmpty() ? prior : prior.concat(" "); - prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); - } - ChangeLogEntry change = new ChangeLogEntry(prior, - reason, - user.getId(), - timestamp - ); - - // create the changelog field in additional info if it does not already exist - OverwrittenData.createChangeLog(pendingObservation); - - // add a new entry to the changelog - pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); - } - - // preview case where observation has already been committed and import ObsVar data is the - // same as has been committed prior to import - } else if (isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { - BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); - existingObs.setObservationVariableName(phenoCol.name()); - observationByHash.get(importHash).setState(ImportObjectState.EXISTING); - observationByHash.get(importHash).setBrAPIObject(existingObs); - - // preview case where observation has already been committed and import ObsVar data is empty prior to import - } else if (!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { - observationByHash.get(importHash).setState(ImportObjectState.EXISTING); - } else { - validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); - - //Timestamp validation - if (timeStampColByPheno.containsKey(phenoCol.name())) { - Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); - validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); - } - } - - } +// private void validateObservation(String importHash) { +// +// +// String importObsValue = phenoCol.getString(rowNum); +// +// +// // error if import observation data already exists and user has not selected to overwrite +// if (commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && +// this.existingObsByObsHash.containsKey(importHash) && +// StringUtils.isNotBlank(phenoCol.getString(rowNum)) && +// !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { +// addRowError( +// phenoCol.name(), +// String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), +// validationErrors, rowNum +// ); +// +// // preview case where observation has already been committed and the import row ObsVar data differs from what +// // had been saved prior to import +// } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { +// +// // add a change log entry when updating the value of an observation +// if (commit) { +// BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); +// DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); +// String timestamp = formatter.format(OffsetDateTime.now()); +// String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; +// String prior = ""; +// if (isValueMatched(importHash, importObsValue)) { +// prior.concat(existingObsByObsHash.get(importHash).getValue()); +// } +// if (timeStampColByPheno.containsKey(phenoCol.name()) && isTimestampMatched(importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { +// prior = prior.isEmpty() ? prior : prior.concat(" "); +// prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); +// } +// ChangeLogEntry change = new ChangeLogEntry(prior, +// reason, +// user.getId(), +// timestamp +// ); +// +// // create the changelog field in additional info if it does not already exist +// OverwrittenData.createChangeLog(pendingObservation); +// +// // add a new entry to the changelog +// pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); +// } +// +// // preview case where observation has already been committed and import ObsVar data is the +// // same as has been committed prior to import +// } else if (isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { +// BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); +// existingObs.setObservationVariableName(phenoCol.name()); +// observationByHash.get(importHash).setState(ImportObjectState.EXISTING); +// observationByHash.get(importHash).setBrAPIObject(existingObs); +// +// // preview case where observation has already been committed and import ObsVar data is empty prior to import +// } else if (!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { +// observationByHash.get(importHash).setState(ImportObjectState.EXISTING); +// } else { +// validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); +// +// //Timestamp validation +// if (timeStampColByPheno.containsKey(phenoCol.name())) { +// Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); +// validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); +// } +// } +// +// } // TODO: used by both workflows - private void updateObservationDependencyValues(Program program) { - String programKey = program.getKey(); - - // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds - this.observationUnitByNameNoScope.values().stream() - .map(PendingImportObject::getBrAPIObject) - .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); - - // Update ObservationVariable DbIds - List traits = getTraitList(program); - CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); - for ( Trait trait: traits) { - traitMap.put(trait.getObservationVariableName(),trait); - } - for (PendingImportObject observation : this.observationByHash.values()) { - String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); - if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { - String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); - observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); - } - } - } +// private void updateObservationDependencyValues(Program program) { +// String programKey = program.getKey(); +// +// // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds +// this.observationUnitByNameNoScope.values().stream() +// .map(PendingImportObject::getBrAPIObject) +// .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); +// +// // Update ObservationVariable DbIds +// List traits = getTraitList(program); +// CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); +// for ( Trait trait: traits) { +// traitMap.put(trait.getObservationVariableName(),trait); +// } +// for (PendingImportObject observation : this.observationByHash.values()) { +// String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); +// if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { +// String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); +// observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); +// } +// } +// } // TODO: used by both workflows - public List commitNewPendingObservationsToBrAPIStore(ImportContext context, PendingData pendingData) { - // filter out observations with no 'value' so they will not be saved - List newObservations = ProcessorData.getNewObjects(this.observationByHash) - .stream() - .filter(obs -> !obs.getValue().isBlank()) - .collect(Collectors.toList()); - - updateObservationDependencyValues(program); - return brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); - - } +// public List commitNewPendingObservationsToBrAPIStore(ImportContext context, PendingData pendingData) { +// // filter out observations with no 'value' so they will not be saved +// List newObservations = ProcessorData.getNewObjects(this.observationByHash) +// .stream() +// .filter(obs -> !obs.getValue().isBlank()) +// .collect(Collectors.toList()); +// +// updateObservationDependencyValues(program); +// return brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); +// +// } // TODO: used by both workflows - public List commitUpdatedPendingObservationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { - List updatedObservations = new ArrayList<>(); - Map mutatedObservationByDbId = ProcessorData - .getMutationsByObjectId(observationByHash, BrAPIObservation::getObservationDbId); - - for (Map.Entry entry : mutatedObservationByDbId.entrySet()) { - String id = entry.getKey(); - BrAPIObservation observation = entry.getValue(); - try { - if (observation == null) { - throw new Exception("Null observation"); - } - BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); - updatedObservations.add(updatedObs); - - if (updatedObs == null) { - throw new Exception("Null updated observation"); - } - - if (!Objects.equals(observation.getValue(), updatedObs.getValue()) - || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { - String message; - if (!Objects.equals(observation.getValue(), updatedObs.getValue())) { - message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); - } else { - message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); - } - throw new Exception(message); - } - - } catch (ApiException e) { - log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException("Error saving experiment import", e); - } catch (Exception e) { - log.error("Error updating observation: ", e); - throw new InternalServerException(e.getMessage(), e); - } - } - - return updatedObservations; - } +// public List commitUpdatedPendingObservationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { +// List updatedObservations = new ArrayList<>(); +// Map mutatedObservationByDbId = ProcessorData +// .getMutationsByObjectId(observationByHash, BrAPIObservation::getObservationDbId); +// +// for (Map.Entry entry : mutatedObservationByDbId.entrySet()) { +// String id = entry.getKey(); +// BrAPIObservation observation = entry.getValue(); +// try { +// if (observation == null) { +// throw new Exception("Null observation"); +// } +// BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); +// updatedObservations.add(updatedObs); +// +// if (updatedObs == null) { +// throw new Exception("Null updated observation"); +// } +// +// if (!Objects.equals(observation.getValue(), updatedObs.getValue()) +// || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { +// String message; +// if (!Objects.equals(observation.getValue(), updatedObs.getValue())) { +// message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); +// } else { +// message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); +// } +// throw new Exception(message); +// } +// +// } catch (ApiException e) { +// log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); +// throw new InternalServerException("Error saving experiment import", e); +// } catch (Exception e) { +// log.error("Error updating observation: ", e); +// throw new InternalServerException(e.getMessage(), e); +// } +// } +// +// return updatedObservations; +// } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index dd8bc2856..708b2b481 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -184,99 +184,99 @@ public List collectMissingOUIds(Set referenceIds, List fetchOrCreateObsUnitPIO(ImportContext importContext, - PendingData pendingData, - ExpUnitContext expUnitContext, - String envSeqValue) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { - PendingImportObject pio; - String key = createObservationUnitKey(importRow); - if (hasAllReferenceUnitIds) { - pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); - } else if (observationUnitByNameNoScope.containsKey(key)) { - pio = observationUnitByNameNoScope.get(key); - } else { - String germplasmName = ""; - if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { - germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) - .getBrAPIObject() - .getGermplasmName(); - } - PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; - UUID trialID = trialPIO.getId(); - UUID datasetId = null; - if (commit) { - datasetId = UUID.fromString(trialPIO.getBrAPIObject() - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); - } - PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); - UUID studyID = studyPIO.getId(); - UUID id = UUID.randomUUID(); - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); - - // check for existing units if this is an existing study - if (studyPIO.getBrAPIObject().getStudyDbId() != null) { - List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); - List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); - if (matchingOU.isEmpty()) { - throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); - } else { - pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); - } - } else { - pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); - } - this.observationUnitByNameNoScope.put(key, pio); - } - return pio; - } +// public PendingImportObject fetchOrCreateObsUnitPIO(ImportContext importContext, +// PendingData pendingData, +// ExpUnitContext expUnitContext, +// String envSeqValue) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { +// PendingImportObject pio; +// String key = createObservationUnitKey(importRow); +// if (hasAllReferenceUnitIds) { +// pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); +// } else if (observationUnitByNameNoScope.containsKey(key)) { +// pio = observationUnitByNameNoScope.get(key); +// } else { +// String germplasmName = ""; +// if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { +// germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) +// .getBrAPIObject() +// .getGermplasmName(); +// } +// PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; +// UUID trialID = trialPIO.getId(); +// UUID datasetId = null; +// if (commit) { +// datasetId = UUID.fromString(trialPIO.getBrAPIObject() +// .getAdditionalInfo().getAsJsonObject() +// .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); +// } +// PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); +// UUID studyID = studyPIO.getId(); +// UUID id = UUID.randomUUID(); +// BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); +// +// // check for existing units if this is an existing study +// if (studyPIO.getBrAPIObject().getStudyDbId() != null) { +// List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); +// List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); +// if (matchingOU.isEmpty()) { +// throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); +// } else { +// pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); +// } +// } else { +// pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); +// } +// this.observationUnitByNameNoScope.put(key, pio); +// } +// return pio; +// } // TODO: used by create workflow - public PendingImportObject fetchOrCreateObsUnitPIO(ImportContext importContext, - PendingData pendingData, - String envSeqValue) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { - PendingImportObject pio; - String key = createObservationUnitKey(importRow); - if (hasAllReferenceUnitIds) { - pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); - } else if (observationUnitByNameNoScope.containsKey(key)) { - pio = observationUnitByNameNoScope.get(key); - } else { - String germplasmName = ""; - if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { - germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) - .getBrAPIObject() - .getGermplasmName(); - } - PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; - UUID trialID = trialPIO.getId(); - UUID datasetId = null; - if (commit) { - datasetId = UUID.fromString(trialPIO.getBrAPIObject() - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); - } - PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); - UUID studyID = studyPIO.getId(); - UUID id = UUID.randomUUID(); - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); - - // check for existing units if this is an existing study - if (studyPIO.getBrAPIObject().getStudyDbId() != null) { - List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); - List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); - if (matchingOU.isEmpty()) { - throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); - } else { - pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); - } - } else { - pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); - } - this.observationUnitByNameNoScope.put(key, pio); - } - return pio; - } +// public PendingImportObject fetchOrCreateObsUnitPIO(ImportContext importContext, +// PendingData pendingData, +// String envSeqValue) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { +// PendingImportObject pio; +// String key = createObservationUnitKey(importRow); +// if (hasAllReferenceUnitIds) { +// pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); +// } else if (observationUnitByNameNoScope.containsKey(key)) { +// pio = observationUnitByNameNoScope.get(key); +// } else { +// String germplasmName = ""; +// if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { +// germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) +// .getBrAPIObject() +// .getGermplasmName(); +// } +// PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; +// UUID trialID = trialPIO.getId(); +// UUID datasetId = null; +// if (commit) { +// datasetId = UUID.fromString(trialPIO.getBrAPIObject() +// .getAdditionalInfo().getAsJsonObject() +// .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); +// } +// PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); +// UUID studyID = studyPIO.getId(); +// UUID id = UUID.randomUUID(); +// BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); +// +// // check for existing units if this is an existing study +// if (studyPIO.getBrAPIObject().getStudyDbId() != null) { +// List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); +// List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); +// if (matchingOU.isEmpty()) { +// throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); +// } else { +// pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); +// } +// } else { +// pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); +// } +// this.observationUnitByNameNoScope.put(key, pio); +// } +// return pio; +// } // TODO: used by both workflows public String createObservationUnitKey(ExperimentObservation importRow) { @@ -289,91 +289,91 @@ public String createObservationUnitKey(String studyName, String obsUnitName) { } // TODO: used by create workflow - public void validateObservationUnits(ValidationErrors validationErrors, - Set uniqueStudyAndObsUnit, - int rowNum, - ExperimentObservation importRow) { - validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); - - String key = createObservationUnitKey(importRow); - PendingImportObject ouPIO = observationUnitByNameNoScope.get(key); - if(ouPIO.getState() == ImportObjectState.NEW && StringUtils.isNotBlank(importRow.getObsUnitID())) { - addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "Could not find observation unit by ObsUnitDBID", validationErrors, rowNum); - } - - validateGeoCoordinates(validationErrors, rowNum, importRow); - } +// public void validateObservationUnits(ValidationErrors validationErrors, +// Set uniqueStudyAndObsUnit, +// int rowNum, +// ExperimentObservation importRow) { +// validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); +// +// String key = createObservationUnitKey(importRow); +// PendingImportObject ouPIO = observationUnitByNameNoScope.get(key); +// if(ouPIO.getState() == ImportObjectState.NEW && StringUtils.isNotBlank(importRow.getObsUnitID())) { +// addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "Could not find observation unit by ObsUnitDBID", validationErrors, rowNum); +// } +// +// validateGeoCoordinates(validationErrors, rowNum, importRow); +// } // TODO: used by both workflows - private void updateObsUnitDependencyValues(String programKey) { - - // update study DbIds - this.studyByNameNoScope.values() - .stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(study -> updateStudyDbId(study, programKey)); - - // update germplasm DbIds - this.existingGermplasmByGID.values() - .stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(this::updateGermplasmDbId); - } +// private void updateObsUnitDependencyValues(String programKey) { +// +// // update study DbIds +// this.studyByNameNoScope.values() +// .stream() +// .filter(Objects::nonNull) +// .distinct() +// .map(PendingImportObject::getBrAPIObject) +// .forEach(study -> updateStudyDbId(study, programKey)); +// +// // update germplasm DbIds +// this.existingGermplasmByGID.values() +// .stream() +// .filter(Objects::nonNull) +// .distinct() +// .map(PendingImportObject::getBrAPIObject) +// .forEach(this::updateGermplasmDbId); +// } // TODO: used by both workflows - public List commitNewPendingObservationUnitsToBrAPIStore(ImportContext context, PendingData pendingData) { - List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); - updateObsUnitDependencyValues(program.getKey()); - List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); - - // set the DbId to the for each newly created Observation Unit - for (BrAPIObservationUnit createdObservationUnit : createdObservationUnits) { - // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope - String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getStudyName(), program.getKey()); - String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getObservationUnitName(), program.getKey()); - String createdObsUnit_key = createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); - this.observationUnitByNameNoScope.get(createdObsUnit_key) - .getBrAPIObject() - .setObservationUnitDbId(createdObservationUnit.getObservationUnitDbId()); - } - - return createdObservationUnits; - } +// public List commitNewPendingObservationUnitsToBrAPIStore(ImportContext context, PendingData pendingData) { +// List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); +// updateObsUnitDependencyValues(program.getKey()); +// List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); +// +// // set the DbId to the for each newly created Observation Unit +// for (BrAPIObservationUnit createdObservationUnit : createdObservationUnits) { +// // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope +// String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getStudyName(), program.getKey()); +// String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getObservationUnitName(), program.getKey()); +// String createdObsUnit_key = createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); +// this.observationUnitByNameNoScope.get(createdObsUnit_key) +// .getBrAPIObject() +// .setObservationUnitDbId(createdObservationUnit.getObservationUnitDbId()); +// } +// +// return createdObservationUnits; +// } // TODO: used by both workflows - public List commitUpdatedPendingObservationUnitToBrAPIStore(ImportContext importContext, PendingData pendingData) { - List updatedUnits = new ArrayList<>(); - - return updatedUnits; - } - - private void updateStudyDbId(BrAPIStudy study, String programKey) { - this.observationUnitByNameNoScope.values() - .stream() - .filter(obsUnit -> obsUnit.getBrAPIObject() - .getStudyName() - .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), programKey))) - .forEach(obsUnit -> { - obsUnit.getBrAPIObject() - .setStudyDbId(study.getStudyDbId()); - obsUnit.getBrAPIObject() - .setTrialDbId(study.getTrialDbId()); - }); - } - - private void updateGermplasmDbId(BrAPIGermplasm germplasm) { - this.observationUnitByNameNoScope.values() - .stream() - .filter(obsUnit -> germplasm.getAccessionNumber() != null && - germplasm.getAccessionNumber().equals(obsUnit - .getBrAPIObject() - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.GID).getAsString())) - .forEach(obsUnit -> obsUnit.getBrAPIObject() - .setGermplasmDbId(germplasm.getGermplasmDbId())); - } +// public List commitUpdatedPendingObservationUnitToBrAPIStore(ImportContext importContext, PendingData pendingData) { +// List updatedUnits = new ArrayList<>(); +// +// return updatedUnits; +// } +// +// private void updateStudyDbId(BrAPIStudy study, String programKey) { +// this.observationUnitByNameNoScope.values() +// .stream() +// .filter(obsUnit -> obsUnit.getBrAPIObject() +// .getStudyName() +// .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), programKey))) +// .forEach(obsUnit -> { +// obsUnit.getBrAPIObject() +// .setStudyDbId(study.getStudyDbId()); +// obsUnit.getBrAPIObject() +// .setTrialDbId(study.getTrialDbId()); +// }); +// } + +// private void updateGermplasmDbId(BrAPIGermplasm germplasm) { +// this.observationUnitByNameNoScope.values() +// .stream() +// .filter(obsUnit -> germplasm.getAccessionNumber() != null && +// germplasm.getAccessionNumber().equals(obsUnit +// .getBrAPIObject() +// .getAdditionalInfo().getAsJsonObject() +// .get(BrAPIAdditionalInfoFields.GID).getAsString())) +// .forEach(obsUnit -> obsUnit.getBrAPIObject() +// .setGermplasmDbId(germplasm.getGermplasmDbId())); +// } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java index 1671d77ee..fdf692df9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java @@ -89,194 +89,194 @@ public Optional> validateMatchedTimestamps(Set obs return ve; } // TODO: used with expUnit workflow - public void validateObservations(PendingData pendingData, - int rowNum, - ImportContext importContext, - ExpUnitContext expUnitContext, - List> phenotypeCols, - CaseInsensitiveMap colVarMap) { - for (Column phenoCol : phenotypeCols) { - String importHash; - String importObsValue = phenoCol.getString(rowNum); - - importHash = getImportObservationHash( - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), - getVariableNameFromColumn(phenoCol), - pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() - ); - - validateObservation(importHash); - } - } +// public void validateObservations(PendingData pendingData, +// int rowNum, +// ImportContext importContext, +// ExpUnitContext expUnitContext, +// List> phenotypeCols, +// CaseInsensitiveMap colVarMap) { +// for (Column phenoCol : phenotypeCols) { +// String importHash; +// String importObsValue = phenoCol.getString(rowNum); +// +// importHash = getImportObservationHash( +// pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), +// getVariableNameFromColumn(phenoCol), +// pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() +// ); +// +// validateObservation(importHash); +// } +// } // TODO: used with create workflow - public void validateObservations(PendingData pendingData, - int rowNum, - ImportContext importContext, - List> phenotypeCols, - CaseInsensitiveMap colVarMap) { - - - for (Column phenoCol : phenotypeCols) { - String importHash; - String importObsValue = phenoCol.getString(rowNum); - - importHash = getImportObservationHash(importRow, phenoCol.name()); - - validateObservation(importHash); - } - - } +// public void validateObservations(PendingData pendingData, +// int rowNum, +// ImportContext importContext, +// List> phenotypeCols, +// CaseInsensitiveMap colVarMap) { +// +// +// for (Column phenoCol : phenotypeCols) { +// String importHash; +// String importObsValue = phenoCol.getString(rowNum); +// +// importHash = getImportObservationHash(importRow, phenoCol.name()); +// +// validateObservation(importHash); +// } +// +// } // TODO: used by both workflows - private void validateObservation(String importHash) { - - - String importObsValue = phenoCol.getString(rowNum); - - - // error if import observation data already exists and user has not selected to overwrite - if (commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && - this.existingObsByObsHash.containsKey(importHash) && - StringUtils.isNotBlank(phenoCol.getString(rowNum)) && - !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { - addRowError( - phenoCol.name(), - String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), - validationErrors, rowNum - ); - - // preview case where observation has already been committed and the import row ObsVar data differs from what - // had been saved prior to import - } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { - - // add a change log entry when updating the value of an observation - if (commit) { - BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); - String timestamp = formatter.format(OffsetDateTime.now()); - String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; - String prior = ""; - if (isValueMatched(importHash, importObsValue)) { - prior.concat(existingObsByObsHash.get(importHash).getValue()); - } - if (timeStampColByPheno.containsKey(phenoCol.name()) && isTimestampMatched(importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { - prior = prior.isEmpty() ? prior : prior.concat(" "); - prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); - } - ChangeLogEntry change = new ChangeLogEntry(prior, - reason, - user.getId(), - timestamp - ); - - // create the changelog field in additional info if it does not already exist - OverwrittenData.createChangeLog(pendingObservation); - - // add a new entry to the changelog - pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); - } - - // preview case where observation has already been committed and import ObsVar data is the - // same as has been committed prior to import - } else if (isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { - BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); - existingObs.setObservationVariableName(phenoCol.name()); - observationByHash.get(importHash).setState(ImportObjectState.EXISTING); - observationByHash.get(importHash).setBrAPIObject(existingObs); - - // preview case where observation has already been committed and import ObsVar data is empty prior to import - } else if (!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { - observationByHash.get(importHash).setState(ImportObjectState.EXISTING); - } else { - validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); - - //Timestamp validation - if (timeStampColByPheno.containsKey(phenoCol.name())) { - Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); - validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); - } - } - - } +// private void validateObservation(String importHash) { +// +// +// String importObsValue = phenoCol.getString(rowNum); +// +// +// // error if import observation data already exists and user has not selected to overwrite +// if (commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && +// this.existingObsByObsHash.containsKey(importHash) && +// StringUtils.isNotBlank(phenoCol.getString(rowNum)) && +// !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { +// addRowError( +// phenoCol.name(), +// String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), +// validationErrors, rowNum +// ); +// +// // preview case where observation has already been committed and the import row ObsVar data differs from what +// // had been saved prior to import +// } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { +// +// // add a change log entry when updating the value of an observation +// if (commit) { +// BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); +// DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); +// String timestamp = formatter.format(OffsetDateTime.now()); +// String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; +// String prior = ""; +// if (isValueMatched(importHash, importObsValue)) { +// prior.concat(existingObsByObsHash.get(importHash).getValue()); +// } +// if (timeStampColByPheno.containsKey(phenoCol.name()) && isTimestampMatched(importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { +// prior = prior.isEmpty() ? prior : prior.concat(" "); +// prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); +// } +// ChangeLogEntry change = new ChangeLogEntry(prior, +// reason, +// user.getId(), +// timestamp +// ); +// +// // create the changelog field in additional info if it does not already exist +// OverwrittenData.createChangeLog(pendingObservation); +// +// // add a new entry to the changelog +// pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); +// } +// +// // preview case where observation has already been committed and import ObsVar data is the +// // same as has been committed prior to import +// } else if (isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { +// BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); +// existingObs.setObservationVariableName(phenoCol.name()); +// observationByHash.get(importHash).setState(ImportObjectState.EXISTING); +// observationByHash.get(importHash).setBrAPIObject(existingObs); +// +// // preview case where observation has already been committed and import ObsVar data is empty prior to import +// } else if (!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { +// observationByHash.get(importHash).setState(ImportObjectState.EXISTING); +// } else { +// validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); +// +// //Timestamp validation +// if (timeStampColByPheno.containsKey(phenoCol.name())) { +// Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); +// validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); +// } +// } +// +// } // TODO: used by both workflows - private void updateObservationDependencyValues(Program program) { - String programKey = program.getKey(); - - // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds - this.observationUnitByNameNoScope.values().stream() - .map(PendingImportObject::getBrAPIObject) - .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); - - // Update ObservationVariable DbIds - List traits = getTraitList(program); - CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); - for ( Trait trait: traits) { - traitMap.put(trait.getObservationVariableName(),trait); - } - for (PendingImportObject observation : this.observationByHash.values()) { - String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); - if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { - String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); - observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); - } - } - } +// private void updateObservationDependencyValues(Program program) { +// String programKey = program.getKey(); +// +// // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds +// this.observationUnitByNameNoScope.values().stream() +// .map(PendingImportObject::getBrAPIObject) +// .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); +// +// // Update ObservationVariable DbIds +// List traits = getTraitList(program); +// CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); +// for ( Trait trait: traits) { +// traitMap.put(trait.getObservationVariableName(),trait); +// } +// for (PendingImportObject observation : this.observationByHash.values()) { +// String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); +// if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { +// String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); +// observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); +// } +// } +// } // TODO: used by both workflows - public List commitNewPendingObservationsToBrAPIStore(ImportContext context, PendingData pendingData) { - // filter out observations with no 'value' so they will not be saved - List newObservations = ProcessorData.getNewObjects(this.observationByHash) - .stream() - .filter(obs -> !obs.getValue().isBlank()) - .collect(Collectors.toList()); - - updateObservationDependencyValues(program); - return brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); - - } +// public List commitNewPendingObservationsToBrAPIStore(ImportContext context, PendingData pendingData) { +// // filter out observations with no 'value' so they will not be saved +// List newObservations = ProcessorData.getNewObjects(this.observationByHash) +// .stream() +// .filter(obs -> !obs.getValue().isBlank()) +// .collect(Collectors.toList()); +// +// updateObservationDependencyValues(program); +// return brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); +// +// } // TODO: used by both workflows - public List commitUpdatedPendingObservationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { - List updatedObservations = new ArrayList<>(); - Map mutatedObservationByDbId = ProcessorData - .getMutationsByObjectId(observationByHash, BrAPIObservation::getObservationDbId); - - for (Map.Entry entry : mutatedObservationByDbId.entrySet()) { - String id = entry.getKey(); - BrAPIObservation observation = entry.getValue(); - try { - if (observation == null) { - throw new Exception("Null observation"); - } - BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); - updatedObservations.add(updatedObs); - - if (updatedObs == null) { - throw new Exception("Null updated observation"); - } - - if (!Objects.equals(observation.getValue(), updatedObs.getValue()) - || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { - String message; - if (!Objects.equals(observation.getValue(), updatedObs.getValue())) { - message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); - } else { - message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); - } - throw new Exception(message); - } - - } catch (ApiException e) { - log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException("Error saving experiment import", e); - } catch (Exception e) { - log.error("Error updating observation: ", e); - throw new InternalServerException(e.getMessage(), e); - } - } - - return updatedObservations; - } +// public List commitUpdatedPendingObservationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { +// List updatedObservations = new ArrayList<>(); +// Map mutatedObservationByDbId = ProcessorData +// .getMutationsByObjectId(observationByHash, BrAPIObservation::getObservationDbId); +// +// for (Map.Entry entry : mutatedObservationByDbId.entrySet()) { +// String id = entry.getKey(); +// BrAPIObservation observation = entry.getValue(); +// try { +// if (observation == null) { +// throw new Exception("Null observation"); +// } +// BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); +// updatedObservations.add(updatedObs); +// +// if (updatedObs == null) { +// throw new Exception("Null updated observation"); +// } +// +// if (!Objects.equals(observation.getValue(), updatedObs.getValue()) +// || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { +// String message; +// if (!Objects.equals(observation.getValue(), updatedObs.getValue())) { +// message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); +// } else { +// message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); +// } +// throw new Exception(message); +// } +// +// } catch (ApiException e) { +// log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); +// throw new InternalServerException("Error saving experiment import", e); +// } catch (Exception e) { +// log.error("Error updating observation: ", e); +// throw new InternalServerException(e.getMessage(), e); +// } +// } +// +// return updatedObservations; +// } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StatisticsService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StatisticsService.java index 8b9844cdb..0867cb81e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StatisticsService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StatisticsService.java @@ -14,76 +14,76 @@ public class StatisticsService { // TODO: used by both workflows - public Map generateStatisticsMap(List importRows) { - // Data for stats. - HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names - HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's - HashSet gidCounter = new HashSet<>(); // set of unique GID's - - for (BrAPIImport row : importRows) { - ExperimentObservation importRow = (ExperimentObservation) row; - // Collect date for stats. - addIfNotNull(environmentNameCounter, importRow.getEnv()); - addIfNotNull(obsUnitsIDCounter, createObservationUnitKey(importRow)); - addIfNotNull(gidCounter, importRow.getGid()); - } - - int numNewObservations = Math.toIntExact( - observationByHash.values() - .stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && - !StringUtils.isBlank(preview.getBrAPIObject() - .getValue())) - .count() - ); - - int numExistingObservations = Math.toIntExact( - this.observationByHash.values() - .stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.EXISTING && - !StringUtils.isBlank(preview.getBrAPIObject() - .getValue())) - .count() - ); - - int numMutatedObservations = Math.toIntExact( - this.observationByHash.values() - .stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED && - !StringUtils.isBlank(preview.getBrAPIObject() - .getValue())) - .count() - ); - - - ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() - .newObjectCount(environmentNameCounter.size()) - .build(); - ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() - .newObjectCount(obsUnitsIDCounter.size()) - .build(); - ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() - .newObjectCount(gidCounter.size()) - .build(); - ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder() - .newObjectCount(numNewObservations) - .build(); - ImportPreviewStatistics existingObservationStats = ImportPreviewStatistics.builder() - .newObjectCount(numExistingObservations) - .build(); - ImportPreviewStatistics mutatedObservationStats = ImportPreviewStatistics.builder() - .newObjectCount(numMutatedObservations) - .build(); - - return Map.of( - "Environments", environmentStats, - "Observation_Units", obdUnitStats, - "GIDs", gidStats, - "Observations", observationStats, - "Existing_Observations", existingObservationStats, - "Mutated_Observations", mutatedObservationStats - ); - } +// public Map generateStatisticsMap(List importRows) { +// // Data for stats. +// HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names +// HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's +// HashSet gidCounter = new HashSet<>(); // set of unique GID's +// +// for (BrAPIImport row : importRows) { +// ExperimentObservation importRow = (ExperimentObservation) row; +// // Collect date for stats. +// addIfNotNull(environmentNameCounter, importRow.getEnv()); +// addIfNotNull(obsUnitsIDCounter, createObservationUnitKey(importRow)); +// addIfNotNull(gidCounter, importRow.getGid()); +// } +// +// int numNewObservations = Math.toIntExact( +// observationByHash.values() +// .stream() +// .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && +// !StringUtils.isBlank(preview.getBrAPIObject() +// .getValue())) +// .count() +// ); +// +// int numExistingObservations = Math.toIntExact( +// this.observationByHash.values() +// .stream() +// .filter(preview -> preview != null && preview.getState() == ImportObjectState.EXISTING && +// !StringUtils.isBlank(preview.getBrAPIObject() +// .getValue())) +// .count() +// ); +// +// int numMutatedObservations = Math.toIntExact( +// this.observationByHash.values() +// .stream() +// .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED && +// !StringUtils.isBlank(preview.getBrAPIObject() +// .getValue())) +// .count() +// ); +// +// +// ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() +// .newObjectCount(environmentNameCounter.size()) +// .build(); +// ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() +// .newObjectCount(obsUnitsIDCounter.size()) +// .build(); +// ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() +// .newObjectCount(gidCounter.size()) +// .build(); +// ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder() +// .newObjectCount(numNewObservations) +// .build(); +// ImportPreviewStatistics existingObservationStats = ImportPreviewStatistics.builder() +// .newObjectCount(numExistingObservations) +// .build(); +// ImportPreviewStatistics mutatedObservationStats = ImportPreviewStatistics.builder() +// .newObjectCount(numMutatedObservations) +// .build(); +// +// return Map.of( +// "Environments", environmentStats, +// "Observation_Units", obdUnitStats, +// "GIDs", gidStats, +// "Observations", observationStats, +// "Existing_Observations", existingObservationStats, +// "Mutated_Observations", mutatedObservationStats +// ); +// } // TODO: used by both workflows public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index f6679e947..0a162aba0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -206,187 +206,187 @@ public String getStudyDbIdBelongingToPendingUnit(PendingImportObject> studyByName - ) throws Exception { - Set studyDbIds = observationUnitByNameNoScope.values() - .stream() - .map(pio -> pio.getBrAPIObject() - .getStudyDbId()) - .collect(Collectors.toSet()); - - List studies = fetchStudiesByDbId(studyDbIds, program); - for (BrAPIStudy study : studies) { - processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); - } - } +// private void initializeStudiesForExistingObservationUnits( +// Program program, +// Map> studyByName +// ) throws Exception { +// Set studyDbIds = observationUnitByNameNoScope.values() +// .stream() +// .map(pio -> pio.getBrAPIObject() +// .getStudyDbId()) +// .collect(Collectors.toSet()); +// +// List studies = fetchStudiesByDbId(studyDbIds, program); +// for (BrAPIStudy study : studies) { +// processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); +// } +// } // TODO: used by expunit workflow - public Map> mapPendingStudyByOUId( - String unitId, - BrAPIObservationUnit unit, - Map> studyByName, - Map> studyByOUId, - Program program - ) { - if (unit.getStudyName() != null) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); - studyByOUId.put(unitId, studyByName.get(studyName)); - } else { - throw new IllegalStateException("Observation unit missing study name: " + unitId); - } - - return studyByOUId; - } +// public Map> mapPendingStudyByOUId( +// String unitId, +// BrAPIObservationUnit unit, +// Map> studyByName, +// Map> studyByOUId, +// Program program +// ) { +// if (unit.getStudyName() != null) { +// String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); +// studyByOUId.put(unitId, studyByName.get(studyName)); +// } else { +// throw new IllegalStateException("Observation unit missing study name: " + unitId); +// } +// +// return studyByOUId; +// } // TODO: used by expunit workflow - private PendingImportObject fetchOrCreateStudyPIO( - ImportContext importContext, - ExpUnitContext expUnitContext, - String expSequenceValue, - Supplier envNextVal - ) throws UnprocessableEntityException { - PendingImportObject pio; - if (hasAllReferenceUnitIds) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), - program.getKey() - ); - pio = studyByNameNoScope.get(studyName); - if (!commit){ - addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); - } - } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { - pio = studyByNameNoScope.get(importRow.getEnv()); - if (!commit){ - addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); - } - } else { - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); - UUID trialID = trialPIO.getId(); - UUID id = UUID.randomUUID(); - BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); - newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies - - // It is assumed that the study has only one season, And that the Years and not - // the dbId's are stored in getSeason() list. - String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season - if (commit) { - if(StringUtils.isNotBlank(year)) { - String seasonID = this.yearToSeasonDbId(year, program.getId()); - newStudy.setSeasons(Collections.singletonList(seasonID)); - } - } else { - addYearToStudyAdditionalInfo(program, newStudy, year); - } - - pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); - this.studyByNameNoScope.put(importRow.getEnv(), pio); - } - return pio; - } +// private PendingImportObject fetchOrCreateStudyPIO( +// ImportContext importContext, +// ExpUnitContext expUnitContext, +// String expSequenceValue, +// Supplier envNextVal +// ) throws UnprocessableEntityException { +// PendingImportObject pio; +// if (hasAllReferenceUnitIds) { +// String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( +// pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), +// program.getKey() +// ); +// pio = studyByNameNoScope.get(studyName); +// if (!commit){ +// addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); +// } +// } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { +// pio = studyByNameNoScope.get(importRow.getEnv()); +// if (!commit){ +// addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); +// } +// } else { +// PendingImportObject trialPIO = hasAllReferenceUnitIds ? +// getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); +// UUID trialID = trialPIO.getId(); +// UUID id = UUID.randomUUID(); +// BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); +// newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies +// +// // It is assumed that the study has only one season, And that the Years and not +// // the dbId's are stored in getSeason() list. +// String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season +// if (commit) { +// if(StringUtils.isNotBlank(year)) { +// String seasonID = this.yearToSeasonDbId(year, program.getId()); +// newStudy.setSeasons(Collections.singletonList(seasonID)); +// } +// } else { +// addYearToStudyAdditionalInfo(program, newStudy, year); +// } +// +// pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); +// this.studyByNameNoScope.put(importRow.getEnv(), pio); +// } +// return pio; +// } // TODO: used by create workflow - private PendingImportObject fetchOrCreateStudyPIO( - ImportContext importContext, - String expSequenceValue, - Supplier envNextVal - ) throws UnprocessableEntityException { - PendingImportObject pio; - if (hasAllReferenceUnitIds) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), - program.getKey() - ); - pio = studyByNameNoScope.get(studyName); - if (!commit){ - addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); - } - } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { - pio = studyByNameNoScope.get(importRow.getEnv()); - if (!commit){ - addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); - } - } else { - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); - UUID trialID = trialPIO.getId(); - UUID id = UUID.randomUUID(); - BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); - newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies - - // It is assumed that the study has only one season, And that the Years and not - // the dbId's are stored in getSeason() list. - String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season - if (commit) { - if(StringUtils.isNotBlank(year)) { - String seasonID = this.yearToSeasonDbId(year, program.getId()); - newStudy.setSeasons(Collections.singletonList(seasonID)); - } - } else { - addYearToStudyAdditionalInfo(program, newStudy, year); - } - - pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); - this.studyByNameNoScope.put(importRow.getEnv(), pio); - } - return pio; - } - - private void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { - // update location DbIds in studies for all distinct locations - mappedBrAPIImport.values() - .stream() - .map(PendingImport::getLocation) - .forEach(this::updateStudyLocationDbId); - - // update trial DbIds in studies for all distinct trials - this.trialByNameNoScope.values() - .stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(trial -> this.updateTrialDbId(trial, programKey)); - } - - private void updateStudyLocationDbId(PendingImportObject location) { - this.studyByNameNoScope.values() - .stream() - .filter(study -> location.getId().toString() - .equals(study.getBrAPIObject() - .getLocationDbId())) - .forEach(study -> study.getBrAPIObject() - .setLocationDbId(location.getBrAPIObject().getLocationDbId())); - } - - private void updateTrialDbId(BrAPITrial trial, String programKey) { - this.studyByNameNoScope.values() - .stream() - .filter(study -> study.getBrAPIObject() - .getTrialName() - .equals(Utilities.removeProgramKey(trial.getTrialName(), programKey))) - .forEach(study -> study.getBrAPIObject() - .setTrialDbId(trial.getTrialDbId())); - } +// private PendingImportObject fetchOrCreateStudyPIO( +// ImportContext importContext, +// String expSequenceValue, +// Supplier envNextVal +// ) throws UnprocessableEntityException { +// PendingImportObject pio; +// if (hasAllReferenceUnitIds) { +// String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( +// pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), +// program.getKey() +// ); +// pio = studyByNameNoScope.get(studyName); +// if (!commit){ +// addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); +// } +// } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { +// pio = studyByNameNoScope.get(importRow.getEnv()); +// if (!commit){ +// addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); +// } +// } else { +// PendingImportObject trialPIO = hasAllReferenceUnitIds ? +// getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); +// UUID trialID = trialPIO.getId(); +// UUID id = UUID.randomUUID(); +// BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); +// newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies +// +// // It is assumed that the study has only one season, And that the Years and not +// // the dbId's are stored in getSeason() list. +// String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season +// if (commit) { +// if(StringUtils.isNotBlank(year)) { +// String seasonID = this.yearToSeasonDbId(year, program.getId()); +// newStudy.setSeasons(Collections.singletonList(seasonID)); +// } +// } else { +// addYearToStudyAdditionalInfo(program, newStudy, year); +// } +// +// pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); +// this.studyByNameNoScope.put(importRow.getEnv(), pio); +// } +// return pio; +// } + +// private void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { +// // update location DbIds in studies for all distinct locations +// mappedBrAPIImport.values() +// .stream() +// .map(PendingImport::getLocation) +// .forEach(this::updateStudyLocationDbId); +// +// // update trial DbIds in studies for all distinct trials +// this.trialByNameNoScope.values() +// .stream() +// .filter(Objects::nonNull) +// .distinct() +// .map(PendingImportObject::getBrAPIObject) +// .forEach(trial -> this.updateTrialDbId(trial, programKey)); +// } +// +// private void updateStudyLocationDbId(PendingImportObject location) { +// this.studyByNameNoScope.values() +// .stream() +// .filter(study -> location.getId().toString() +// .equals(study.getBrAPIObject() +// .getLocationDbId())) +// .forEach(study -> study.getBrAPIObject() +// .setLocationDbId(location.getBrAPIObject().getLocationDbId())); +// } +// +// private void updateTrialDbId(BrAPITrial trial, String programKey) { +// this.studyByNameNoScope.values() +// .stream() +// .filter(study -> study.getBrAPIObject() +// .getTrialName() +// .equals(Utilities.removeProgramKey(trial.getTrialName(), programKey))) +// .forEach(study -> study.getBrAPIObject() +// .setTrialDbId(trial.getTrialDbId())); +// } // TODO: used by both workflows - public List commitNewPendingStudiessToBrAPIStore(ImportContext context, PendingData pendingData) { - List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); - updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); - List createdStudies = brAPIStudyDAO.createBrAPIStudies(newStudies, program.getId(), upload); - - // set the DbId to the for each newly created study - for (BrAPIStudy createdStudy : createdStudies) { - String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData(createdStudy.getStudyName(), program.getKey()); - this.studyByNameNoScope.get(createdStudy_name_no_key) - .getBrAPIObject() - .setStudyDbId(createdStudy.getStudyDbId()); - } - - return createdStudies; - } +// public List commitNewPendingStudiessToBrAPIStore(ImportContext context, PendingData pendingData) { +// List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); +// updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); +// List createdStudies = brAPIStudyDAO.createBrAPIStudies(newStudies, program.getId(), upload); +// +// // set the DbId to the for each newly created study +// for (BrAPIStudy createdStudy : createdStudies) { +// String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData(createdStudy.getStudyName(), program.getKey()); +// this.studyByNameNoScope.get(createdStudy_name_no_key) +// .getBrAPIObject() +// .setStudyDbId(createdStudy.getStudyDbId()); +// } +// +// return createdStudies; +// } // TODO: used by both workflows public List commitUpdatedPendingStudiesToBrAPIStore(ImportContext importContext, PendingData pendingData) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index b82c5169d..fe68e61c3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -153,15 +153,6 @@ public List fetchBrapiTrialsByDbId(Set trialDbIds, Program p // TODO: also used in other workflow - - /** - * Initializes trials for existing observation units. - * - * @param program The program object. - * @param observationUnitByNameNoScope A map containing observation units by name (without scope). - * @param trialByName A map containing trials by name. (will be modified in place) - * - */ public void initializeTrialsForExistingObservationUnits(ImportContext importContext, PendingData pendingData) { if(pendingData.getObservationUnitByNameNoScope().size() > 0) { @@ -365,86 +356,86 @@ public Map> mapPendingTrialByOUId( } // TODO: overloaded method used by expunit workflow - public PendingImportObject fetchOrCreateTrialPIO( - ImportContext importContext, - PendingData pendingData, - ExpUnitContext expUnitContext - ) throws UnprocessableEntityException { - PendingImportObject trialPio; - - - - trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); - - - return trialPio; - } +// public PendingImportObject fetchOrCreateTrialPIO( +// ImportContext importContext, +// PendingData pendingData, +// ExpUnitContext expUnitContext +// ) throws UnprocessableEntityException { +// PendingImportObject trialPio; +// +// +// +// trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); +// +// +// return trialPio; +// } // TODO: overloaded method used by create workflow - public PendingImportObject fetchOrCreateTrialPIO( - ImportContext importContext, - PendingData pendingData - ) throws UnprocessableEntityException { - PendingImportObject trialPio; - - - if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { - PendingImportObject envPio; - trialPio = trialByNameNoScope.get(importRow.getExpTitle()); - envPio = studyByNameNoScope.get(importRow.getEnv()); - - // creating new units for existing experiments and environments is not possible - if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && - (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ - throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); - } - } else if (!trialByNameNoScope.isEmpty()) { - throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); - } else { - UUID id = UUID.randomUUID(); - String expSeqValue = null; - if (commit) { - expSeqValue = expNextVal.get().toString(); - } - BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); - trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); - trialByNameNoScope.put(importRow.getExpTitle(), trialPio); - } - - return trialPio; - } +// public PendingImportObject fetchOrCreateTrialPIO( +// ImportContext importContext, +// PendingData pendingData +// ) throws UnprocessableEntityException { +// PendingImportObject trialPio; +// +// +// if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { +// PendingImportObject envPio; +// trialPio = trialByNameNoScope.get(importRow.getExpTitle()); +// envPio = studyByNameNoScope.get(importRow.getEnv()); +// +// // creating new units for existing experiments and environments is not possible +// if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && +// (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ +// throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); +// } +// } else if (!trialByNameNoScope.isEmpty()) { +// throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); +// } else { +// UUID id = UUID.randomUUID(); +// String expSeqValue = null; +// if (commit) { +// expSeqValue = expNextVal.get().toString(); +// } +// BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); +// trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); +// trialByNameNoScope.put(importRow.getExpTitle(), trialPio); +// } +// +// return trialPio; +// } // TODO: used by both workflows - public List commitNewPendingTrialsToBrAPIStore(ImportContext context, PendingData pendingData) { - List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); - List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newTrials, program.getId(), upload)); - // set the DbId to the for each newly created trial - for (BrAPITrial createdTrial : createdTrials) { - String createdTrialName = Utilities.removeProgramKey(createdTrial.getTrialName(), program.getKey()); - this.trialByNameNoScope.get(createdTrialName) - .getBrAPIObject() - .setTrialDbId(createdTrial.getTrialDbId()); - } - return createdTrials; - } - - public List commitUpdatedPendingTrialsToBrAPIStore(ImportContext importContext, PendingData pendingData) { - List updatedTrials = new ArrayList<>(); - Map mutatedTrialsById = ProcessorData - .getMutationsByObjectId(trialByNameNoScope, BrAPITrial::getTrialDbId); - for (Map.Entry entry : mutatedTrialsById.entrySet()) { - String id = entry.getKey(); - BrAPITrial trial = entry.getValue(); - try { - updatedTrials.add(brapiTrialDAO.updateBrAPITrial(id, trial, program.getId())); - } catch (ApiException e) { - log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException("Error saving experiment import", e); - } catch (Exception e) { - log.error("Error updating dataset observation variables: ", e); - throw new InternalServerException(e.getMessage(), e); - } - } - return updatedTrials; - } +// public List commitNewPendingTrialsToBrAPIStore(ImportContext context, PendingData pendingData) { +// List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); +// List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newTrials, program.getId(), upload)); +// // set the DbId to the for each newly created trial +// for (BrAPITrial createdTrial : createdTrials) { +// String createdTrialName = Utilities.removeProgramKey(createdTrial.getTrialName(), program.getKey()); +// this.trialByNameNoScope.get(createdTrialName) +// .getBrAPIObject() +// .setTrialDbId(createdTrial.getTrialDbId()); +// } +// return createdTrials; +// } + +// public List commitUpdatedPendingTrialsToBrAPIStore(ImportContext importContext, PendingData pendingData) { +// List updatedTrials = new ArrayList<>(); +// Map mutatedTrialsById = ProcessorData +// .getMutationsByObjectId(trialByNameNoScope, BrAPITrial::getTrialDbId); +// for (Map.Entry entry : mutatedTrialsById.entrySet()) { +// String id = entry.getKey(); +// BrAPITrial trial = entry.getValue(); +// try { +// updatedTrials.add(brapiTrialDAO.updateBrAPITrial(id, trial, program.getId())); +// } catch (ApiException e) { +// log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); +// throw new InternalServerException("Error saving experiment import", e); +// } catch (Exception e) { +// log.error("Error updating dataset observation variables: ", e); +// throw new InternalServerException(e.getMessage(), e); +// } +// } +// return updatedTrials; +// } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java index dd7bc0af2..ccdae7937 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java @@ -22,291 +22,291 @@ public class ValidateService { // TODO: used by expUnit workflow - public void prepareDataForValidation(ImportContext importContext, - ExpUnitContext expUnitContext, - List> phenotypeCols) { - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); - List> observations = mappedImportRow.getObservations(); - String observationHash; - if (hasAllReferenceUnitIds) { - String refOUId = importRow.getObsUnitID(); - mappedImportRow.setTrial(pendingTrialByOUId.get(refOUId)); - mappedImportRow.setLocation(pendingLocationByOUId.get(refOUId)); - mappedImportRow.setStudy(pendingStudyByOUId.get(refOUId)); - mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(refOUId)); - mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(refOUId)); - - // loop over phenotype column observation data for current row - for (Column column : phenotypeCols) { - observationHash = getObservationHash( - pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + - pendingObsUnitByOUId.get(refOUId).getBrAPIObject().getObservationUnitName(), - getVariableNameFromColumn(column), - pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() - ); - - // if value was blank won't be entry in map for this observation - observations.add(observationByHash.get(observationHash)); - } - - } else { - mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); - mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); - mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); - mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); - mappedImportRow.setGermplasm(getGidPIO(importRow)); - - // loop over phenotype column observation data for current row - for (Column column : phenotypeCols) { - - // if value was blank won't be entry in map for this observation - observations.add(observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); - } - } - - mappedBrAPIImport.put(rowNum, mappedImportRow); - } - } +// public void prepareDataForValidation(ImportContext importContext, +// ExpUnitContext expUnitContext, +// List> phenotypeCols) { +// for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { +// ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); +// PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); +// List> observations = mappedImportRow.getObservations(); +// String observationHash; +// if (hasAllReferenceUnitIds) { +// String refOUId = importRow.getObsUnitID(); +// mappedImportRow.setTrial(pendingTrialByOUId.get(refOUId)); +// mappedImportRow.setLocation(pendingLocationByOUId.get(refOUId)); +// mappedImportRow.setStudy(pendingStudyByOUId.get(refOUId)); +// mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(refOUId)); +// mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(refOUId)); +// +// // loop over phenotype column observation data for current row +// for (Column column : phenotypeCols) { +// observationHash = getObservationHash( +// pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + +// pendingObsUnitByOUId.get(refOUId).getBrAPIObject().getObservationUnitName(), +// getVariableNameFromColumn(column), +// pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() +// ); +// +// // if value was blank won't be entry in map for this observation +// observations.add(observationByHash.get(observationHash)); +// } +// +// } else { +// mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); +// mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); +// mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); +// mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); +// mappedImportRow.setGermplasm(getGidPIO(importRow)); +// +// // loop over phenotype column observation data for current row +// for (Column column : phenotypeCols) { +// +// // if value was blank won't be entry in map for this observation +// observations.add(observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); +// } +// } +// +// mappedBrAPIImport.put(rowNum, mappedImportRow); +// } +// } // TODO: used by create workflow - public void prepareDataForValidation(ImportContext importContext, - List> phenotypeCols) { - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); - List> observations = mappedImportRow.getObservations(); - String observationHash; - if (hasAllReferenceUnitIds) { - String refOUId = importRow.getObsUnitID(); - mappedImportRow.setTrial(pendingTrialByOUId.get(refOUId)); - mappedImportRow.setLocation(pendingLocationByOUId.get(refOUId)); - mappedImportRow.setStudy(pendingStudyByOUId.get(refOUId)); - mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(refOUId)); - mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(refOUId)); - - // loop over phenotype column observation data for current row - for (Column column : phenotypeCols) { - observationHash = getObservationHash( - pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + - pendingObsUnitByOUId.get(refOUId).getBrAPIObject().getObservationUnitName(), - getVariableNameFromColumn(column), - pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() - ); - - // if value was blank won't be entry in map for this observation - observations.add(observationByHash.get(observationHash)); - } - - } else { - mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); - mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); - mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); - mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); - mappedImportRow.setGermplasm(getGidPIO(importRow)); - - // loop over phenotype column observation data for current row - for (Column column : phenotypeCols) { - - // if value was blank won't be entry in map for this observation - observations.add(observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); - } - } - - mappedBrAPIImport.put(rowNum, mappedImportRow); - } - } +// public void prepareDataForValidation(ImportContext importContext, +// List> phenotypeCols) { +// for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { +// ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); +// PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); +// List> observations = mappedImportRow.getObservations(); +// String observationHash; +// if (hasAllReferenceUnitIds) { +// String refOUId = importRow.getObsUnitID(); +// mappedImportRow.setTrial(pendingTrialByOUId.get(refOUId)); +// mappedImportRow.setLocation(pendingLocationByOUId.get(refOUId)); +// mappedImportRow.setStudy(pendingStudyByOUId.get(refOUId)); +// mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(refOUId)); +// mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(refOUId)); +// +// // loop over phenotype column observation data for current row +// for (Column column : phenotypeCols) { +// observationHash = getObservationHash( +// pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + +// pendingObsUnitByOUId.get(refOUId).getBrAPIObject().getObservationUnitName(), +// getVariableNameFromColumn(column), +// pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() +// ); +// +// // if value was blank won't be entry in map for this observation +// observations.add(observationByHash.get(observationHash)); +// } +// +// } else { +// mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); +// mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); +// mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); +// mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); +// mappedImportRow.setGermplasm(getGidPIO(importRow)); +// +// // loop over phenotype column observation data for current row +// for (Column column : phenotypeCols) { +// +// // if value was blank won't be entry in map for this observation +// observations.add(observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); +// } +// } +// +// mappedBrAPIImport.put(rowNum, mappedImportRow); +// } +// } // TODO: used by expUnit workflow - public void validateFields(ImportContext importContext, - PendingData pendingData, - ExpUnitContext expUnitContext, - List referencedTraits, Program program, - List> phenotypeCols) { - //fetching any existing observations for any OUs in the import - CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); - for ( Trait trait: referencedTraits) { - colVarMap.put(trait.getObservationVariableName(),trait); - } - Set uniqueStudyAndObsUnit = new HashSet<>(); - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); - if (hasAllReferenceUnitIds) { - validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); - } else { - if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. - validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); - } - validateTestOrCheck(importRow, validationErrors, rowNum); - validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); - validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); - validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); - } - } - } +// public void validateFields(ImportContext importContext, +// PendingData pendingData, +// ExpUnitContext expUnitContext, +// List referencedTraits, Program program, +// List> phenotypeCols) { +// //fetching any existing observations for any OUs in the import +// CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); +// for ( Trait trait: referencedTraits) { +// colVarMap.put(trait.getObservationVariableName(),trait); +// } +// Set uniqueStudyAndObsUnit = new HashSet<>(); +// for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { +// ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); +// PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); +// if (hasAllReferenceUnitIds) { +// validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); +// } else { +// if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. +// validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); +// } +// validateTestOrCheck(importRow, validationErrors, rowNum); +// validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); +// validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); +// validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); +// } +// } +// } // TODO: used by create workflow - public void validateFields(ImportContext importContext, - PendingData pendingData, - List referencedTraits, Program program, - List> phenotypeCols) { - //fetching any existing observations for any OUs in the import - CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); - for ( Trait trait: referencedTraits) { - colVarMap.put(trait.getObservationVariableName(),trait); - } - Set uniqueStudyAndObsUnit = new HashSet<>(); - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); - if (hasAllReferenceUnitIds) { - validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); - } else { - if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. - validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); - } - validateTestOrCheck(importRow, validationErrors, rowNum); - validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); - validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); - validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); - } - } - } +// public void validateFields(ImportContext importContext, +// PendingData pendingData, +// List referencedTraits, Program program, +// List> phenotypeCols) { +// //fetching any existing observations for any OUs in the import +// CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); +// for ( Trait trait: referencedTraits) { +// colVarMap.put(trait.getObservationVariableName(),trait); +// } +// Set uniqueStudyAndObsUnit = new HashSet<>(); +// for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { +// ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); +// PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); +// if (hasAllReferenceUnitIds) { +// validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); +// } else { +// if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. +// validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); +// } +// validateTestOrCheck(importRow, validationErrors, rowNum); +// validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); +// validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); +// validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); +// } +// } +// } // TODO: used by create workflow - private void validateTestOrCheck(ExperimentObservation importRow, ValidationErrors validationErrors, int rowNum) { - String testOrCheck = importRow.getTestOrCheck(); - if ( ! ( testOrCheck==null || testOrCheck.isBlank() - || "C".equalsIgnoreCase(testOrCheck) || "CHECK".equalsIgnoreCase(testOrCheck) - || "T".equalsIgnoreCase(testOrCheck) || "TEST".equalsIgnoreCase(testOrCheck) ) - ){ - addRowError(ExperimentObservation.Columns.TEST_CHECK, String.format("Invalid value (%s)", testOrCheck), validationErrors, rowNum) ; - } - } +// private void validateTestOrCheck(ExperimentObservation importRow, ValidationErrors validationErrors, int rowNum) { +// String testOrCheck = importRow.getTestOrCheck(); +// if ( ! ( testOrCheck==null || testOrCheck.isBlank() +// || "C".equalsIgnoreCase(testOrCheck) || "CHECK".equalsIgnoreCase(testOrCheck) +// || "T".equalsIgnoreCase(testOrCheck) || "TEST".equalsIgnoreCase(testOrCheck) ) +// ){ +// addRowError(ExperimentObservation.Columns.TEST_CHECK, String.format("Invalid value (%s)", testOrCheck), validationErrors, rowNum) ; +// } +// } // TODO: used by create workflow - private void validateConditionallyRequired(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, Program program, boolean commit) { - ImportObjectState expState = this.trialByNameNoScope.get(importRow.getExpTitle()) - .getState(); - ImportObjectState envState = this.studyByNameNoScope.get(importRow.getEnv()).getState(); - - String errorMessage = BLANK_FIELD_EXPERIMENT; - if (expState == ImportObjectState.EXISTING && envState == ImportObjectState.NEW) { - errorMessage = BLANK_FIELD_ENV; - } else if(expState == ImportObjectState.EXISTING && envState == ImportObjectState.EXISTING) { - errorMessage = BLANK_FIELD_OBS; - } - - if(expState == ImportObjectState.NEW || envState == ImportObjectState.NEW) { - validateRequiredCell(importRow.getGid(), ExperimentObservation.Columns.GERMPLASM_GID, errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getExpTitle(), ExperimentObservation.Columns.EXP_TITLE,errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getExpUnit(), ExperimentObservation.Columns.EXP_UNIT, errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getExpType(), ExperimentObservation.Columns.EXP_TYPE, errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getEnv(), ExperimentObservation.Columns.ENV, errorMessage, validationErrors, rowNum); - if(validateRequiredCell(importRow.getEnvLocation(), ExperimentObservation.Columns.ENV_LOCATION, errorMessage, validationErrors, rowNum)) { - if(!Utilities.removeProgramKeyAndUnknownAdditionalData(this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getLocationName(), program.getKey()).equals(importRow.getEnvLocation())) { - addRowError(ExperimentObservation.Columns.ENV_LOCATION, ENV_LOCATION_MISMATCH, validationErrors, rowNum); - } - } - if(validateRequiredCell(importRow.getEnvYear(), ExperimentObservation.Columns.ENV_YEAR, errorMessage, validationErrors, rowNum)) { - String studyYear = StringUtils.defaultString( this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getSeasons().get(0) ); - String rowYear = importRow.getEnvYear(); - if(commit) { - rowYear = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); - } - if(StringUtils.isNotBlank(studyYear) && !studyYear.equals(rowYear)) { - addRowError(ExperimentObservation.Columns.ENV_YEAR, ENV_YEAR_MISMATCH, validationErrors, rowNum); - } - } - validateRequiredCell(importRow.getExpUnitId(), ExperimentObservation.Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getExpReplicateNo(), ExperimentObservation.Columns.REP_NUM, errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getExpBlockNo(), ExperimentObservation.Columns.BLOCK_NUM, errorMessage, validationErrors, rowNum); - - if(StringUtils.isNotBlank(importRow.getObsUnitID())) { - addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); - } - } else { - //Check if existing environment. If so, ObsUnitId must be assigned - validateRequiredCell( - importRow.getObsUnitID(), - ExperimentObservation.Columns.OBS_UNIT_ID, - MISSING_OBS_UNIT_ID_ERROR, - validationErrors, - rowNum - ); - } - } +// private void validateConditionallyRequired(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, Program program, boolean commit) { +// ImportObjectState expState = this.trialByNameNoScope.get(importRow.getExpTitle()) +// .getState(); +// ImportObjectState envState = this.studyByNameNoScope.get(importRow.getEnv()).getState(); +// +// String errorMessage = BLANK_FIELD_EXPERIMENT; +// if (expState == ImportObjectState.EXISTING && envState == ImportObjectState.NEW) { +// errorMessage = BLANK_FIELD_ENV; +// } else if(expState == ImportObjectState.EXISTING && envState == ImportObjectState.EXISTING) { +// errorMessage = BLANK_FIELD_OBS; +// } +// +// if(expState == ImportObjectState.NEW || envState == ImportObjectState.NEW) { +// validateRequiredCell(importRow.getGid(), ExperimentObservation.Columns.GERMPLASM_GID, errorMessage, validationErrors, rowNum); +// validateRequiredCell(importRow.getExpTitle(), ExperimentObservation.Columns.EXP_TITLE,errorMessage, validationErrors, rowNum); +// validateRequiredCell(importRow.getExpUnit(), ExperimentObservation.Columns.EXP_UNIT, errorMessage, validationErrors, rowNum); +// validateRequiredCell(importRow.getExpType(), ExperimentObservation.Columns.EXP_TYPE, errorMessage, validationErrors, rowNum); +// validateRequiredCell(importRow.getEnv(), ExperimentObservation.Columns.ENV, errorMessage, validationErrors, rowNum); +// if(validateRequiredCell(importRow.getEnvLocation(), ExperimentObservation.Columns.ENV_LOCATION, errorMessage, validationErrors, rowNum)) { +// if(!Utilities.removeProgramKeyAndUnknownAdditionalData(this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getLocationName(), program.getKey()).equals(importRow.getEnvLocation())) { +// addRowError(ExperimentObservation.Columns.ENV_LOCATION, ENV_LOCATION_MISMATCH, validationErrors, rowNum); +// } +// } +// if(validateRequiredCell(importRow.getEnvYear(), ExperimentObservation.Columns.ENV_YEAR, errorMessage, validationErrors, rowNum)) { +// String studyYear = StringUtils.defaultString( this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getSeasons().get(0) ); +// String rowYear = importRow.getEnvYear(); +// if(commit) { +// rowYear = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); +// } +// if(StringUtils.isNotBlank(studyYear) && !studyYear.equals(rowYear)) { +// addRowError(ExperimentObservation.Columns.ENV_YEAR, ENV_YEAR_MISMATCH, validationErrors, rowNum); +// } +// } +// validateRequiredCell(importRow.getExpUnitId(), ExperimentObservation.Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); +// validateRequiredCell(importRow.getExpReplicateNo(), ExperimentObservation.Columns.REP_NUM, errorMessage, validationErrors, rowNum); +// validateRequiredCell(importRow.getExpBlockNo(), ExperimentObservation.Columns.BLOCK_NUM, errorMessage, validationErrors, rowNum); +// +// if(StringUtils.isNotBlank(importRow.getObsUnitID())) { +// addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); +// } +// } else { +// //Check if existing environment. If so, ObsUnitId must be assigned +// validateRequiredCell( +// importRow.getObsUnitID(), +// ExperimentObservation.Columns.OBS_UNIT_ID, +// MISSING_OBS_UNIT_ID_ERROR, +// validationErrors, +// rowNum +// ); +// } +// } // TODO: used by create workflow - public void validateGeoCoordinates(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow) { - - String lat = importRow.getLatitude(); - String lon = importRow.getLongitude(); - String elevation = importRow.getElevation(); - - // If any of Lat, Long, or Elevation are provided, Lat and Long must both be provided. - if (StringUtils.isNotBlank(lat) || StringUtils.isNotBlank(lon) || StringUtils.isNotBlank(elevation)) { - if (StringUtils.isBlank(lat)) { - addRowError(ExperimentObservation.Columns.LAT, "Latitude must be provided for complete coordinate specification", validationErrors, rowNum); - } - if (StringUtils.isBlank(lon)) { - addRowError(ExperimentObservation.Columns.LONG, "Longitude must be provided for complete coordinate specification", validationErrors, rowNum); - } - } - - // Validate coordinate values - boolean latBadValue = false; - boolean lonBadValue = false; - boolean elevationBadValue = false; - double latDouble; - double lonDouble; - double elevationDouble; - - // Only check latitude format if not blank since already had previous error - if (StringUtils.isNotBlank(lat)) { - try { - latDouble = Double.parseDouble(lat); - if (latDouble < -90 || latDouble > 90) { - latBadValue = true; - } - } catch (NumberFormatException e) { - latBadValue = true; - } - } - - // Only check longitude format if not blank since already had previous error - if (StringUtils.isNotBlank(lon)) { - try { - lonDouble = Double.parseDouble(lon); - if (lonDouble < -180 || lonDouble > 180) { - lonBadValue = true; - } - } catch (NumberFormatException e) { - lonBadValue = true; - } - } - - if (StringUtils.isNotBlank(elevation)) { - try { - elevationDouble = Double.parseDouble(elevation); - } catch (NumberFormatException e) { - elevationBadValue = true; - } - } - - if (latBadValue) { - addRowError(ExperimentObservation.Columns.LAT, "Invalid Lat value (expected range -90 to 90)", validationErrors, rowNum); - } - - if (lonBadValue) { - addRowError(ExperimentObservation.Columns.LONG, "Invalid Long value (expected range -180 to 180)", validationErrors, rowNum); - } - - if (elevationBadValue) { - addRowError(ExperimentObservation.Columns.LONG, "Invalid Elevation value (numerals expected)", validationErrors, rowNum); - } - - } +// public void validateGeoCoordinates(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow) { +// +// String lat = importRow.getLatitude(); +// String lon = importRow.getLongitude(); +// String elevation = importRow.getElevation(); +// +// // If any of Lat, Long, or Elevation are provided, Lat and Long must both be provided. +// if (StringUtils.isNotBlank(lat) || StringUtils.isNotBlank(lon) || StringUtils.isNotBlank(elevation)) { +// if (StringUtils.isBlank(lat)) { +// addRowError(ExperimentObservation.Columns.LAT, "Latitude must be provided for complete coordinate specification", validationErrors, rowNum); +// } +// if (StringUtils.isBlank(lon)) { +// addRowError(ExperimentObservation.Columns.LONG, "Longitude must be provided for complete coordinate specification", validationErrors, rowNum); +// } +// } +// +// // Validate coordinate values +// boolean latBadValue = false; +// boolean lonBadValue = false; +// boolean elevationBadValue = false; +// double latDouble; +// double lonDouble; +// double elevationDouble; +// +// // Only check latitude format if not blank since already had previous error +// if (StringUtils.isNotBlank(lat)) { +// try { +// latDouble = Double.parseDouble(lat); +// if (latDouble < -90 || latDouble > 90) { +// latBadValue = true; +// } +// } catch (NumberFormatException e) { +// latBadValue = true; +// } +// } +// +// // Only check longitude format if not blank since already had previous error +// if (StringUtils.isNotBlank(lon)) { +// try { +// lonDouble = Double.parseDouble(lon); +// if (lonDouble < -180 || lonDouble > 180) { +// lonBadValue = true; +// } +// } catch (NumberFormatException e) { +// lonBadValue = true; +// } +// } +// +// if (StringUtils.isNotBlank(elevation)) { +// try { +// elevationDouble = Double.parseDouble(elevation); +// } catch (NumberFormatException e) { +// elevationBadValue = true; +// } +// } +// +// if (latBadValue) { +// addRowError(ExperimentObservation.Columns.LAT, "Invalid Lat value (expected range -90 to 90)", validationErrors, rowNum); +// } +// +// if (lonBadValue) { +// addRowError(ExperimentObservation.Columns.LONG, "Invalid Long value (expected range -180 to 180)", validationErrors, rowNum); +// } +// +// if (elevationBadValue) { +// addRowError(ExperimentObservation.Columns.LONG, "Invalid Elevation value (numerals expected)", validationErrors, rowNum); +// } +// +// } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java index 7f0b741fe..0d960b4d3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java @@ -63,7 +63,7 @@ public Workflow getWorkflow(ImportContext context) { } else { // No ObsUnitIDs so creating experiment or appending env - return createNewExperimentWorkflowProvider.get(); + return (Workflow) createNewExperimentWorkflowProvider.get(); // TODO: different workflow for appending envs? Would have a dependency on DAO to check for existing trial name } } From 5564ca1e97a7509d0ef6edc353462471ee61ee47 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 22 May 2024 18:17:05 -0400 Subject: [PATCH 066/203] create micronaut composite pattern classes for experiment import workflow --- .../model/imports/ImportServiceContext.java | 1 + .../ExperimentImportService.java | 27 ++++----- .../importer/model/workflow/Action.java | 29 +++++++++ .../model/workflow/ImportWorkflow.java | 12 ++++ .../model/workflow/ImportWorkflowResult.java | 33 +++++++++++ .../importer/services/FileImportService.java | 10 ++-- .../experiment/ExperimentImportWorkflow.java | 59 +++++++++++++++++++ .../AppendOverwriteObservationWorkflow.java | 53 +++++++++++++++++ .../workflow/NewExperimentWorkflow.java | 54 +++++++++++++++++ .../workflow/NewEnvironmentWorkflow.java | 53 +++++++++++++++++ 10 files changed, 309 insertions(+), 22 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java index 45393f67c..c5a479e90 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -32,6 +32,7 @@ @AllArgsConstructor @NoArgsConstructor public class ImportServiceContext { + private String action; private Workflow workflow; private List brAPIImports; private Table data; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index c6f68b251..de7d54961 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -23,9 +23,11 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import tech.tablesaw.api.Table; @@ -34,6 +36,7 @@ import javax.inject.Provider; import javax.inject.Singleton; import java.util.List; +import java.util.Optional; @Singleton @Slf4j @@ -43,12 +46,16 @@ public class ExperimentImportService implements BrAPIImportService { private final Provider experimentProcessorProvider; private final Provider processorManagerProvider; + private final ExperimentImportWorkflow workflow; @Inject - public ExperimentImportService(Provider experimentProcessorProvider, Provider processorManagerProvider) + public ExperimentImportService(Provider experimentProcessorProvider, + Provider processorManagerProvider, + ExperimentImportWorkflow workflow) { this.experimentProcessorProvider = experimentProcessorProvider; this.processorManagerProvider = processorManagerProvider; + this.workflow = workflow; } @Override @@ -70,23 +77,11 @@ public String getMissingColumnMsg(String columnName) { public ImportPreviewResponse process(ImportServiceContext context) throws Exception { - ImportPreviewResponse response = null; - List processors = List.of(experimentProcessorProvider.get()); - - if (context.getWorkflow() != null) { - log.info("Workflow: " + context.getWorkflow().getName()); + if (!context.getAction().isEmpty()) { + log.info("Workflow: " + context.getAction()); } - // TODO: change to calling workflow process instead of processor manager - response = processorManagerProvider.get().process(context.getBrAPIImports(), - processors, - context.getData(), - context.getProgram(), - context.getUpload(), - context.getUser(), - context.isCommit()); - return response; - + return workflow.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java new file mode 100644 index 000000000..65c3d0cbb --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java @@ -0,0 +1,29 @@ +///* +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; + +@Getter +@Setter +@ToString +@AllArgsConstructor +public class Action { + private String name; + private int order; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java new file mode 100644 index 000000000..c5dfc1957 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java @@ -0,0 +1,12 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +import io.micronaut.core.order.Ordered; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; + +import java.util.Optional; + +@FunctionalInterface +public interface ImportWorkflow extends Ordered { + Optional process(ImportServiceContext context); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java new file mode 100644 index 000000000..2e61a6d46 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java @@ -0,0 +1,33 @@ +///* +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; + +import java.util.Optional; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +public class ImportWorkflowResult { + private Action action; + private Optional importPreviewResponse; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index ef6c5f884..59438c696 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -435,14 +435,12 @@ private void processFile(UUID workflowId, List finalBrAPIImportList // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { - Workflow workflow = null; - if (workflowId != null) { - Optional optionalWorkflow = workflowFactory.getWorkflow(workflowId); - workflow = optionalWorkflow.orElse(null); - } + + final Workflow[] workflow = {null}; + Optional.ofNullable(workflowId).ifPresent(id -> workflow[0] = workflowFactory.getWorkflow(id).orElse(null)); ImportServiceContext context = ImportServiceContext.builder() - .workflow(workflow) + .workflow(workflow[0]) .brAPIImports(finalBrAPIImportList) .data(data) .program(program) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java new file mode 100644 index 000000000..e8a7d45da --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java @@ -0,0 +1,59 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment; + +import io.micronaut.context.annotation.Primary; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; + +import javax.inject.Singleton; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +@Primary +@Singleton +public class ExperimentImportWorkflow implements ImportWorkflow { + private final List workflows; + + public ExperimentImportWorkflow(List workflows) { + this.workflows = workflows; + } + + @Override + public Optional process(ImportServiceContext context) { + return workflows.stream() + .map(workflow->workflow.process(context)) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst(); + } + public List getWorkflows() { + return workflows.stream() + .map(workflow->workflow.process(null)) + .filter(Optional::isPresent) + .map(Optional::get) + .map(result->result.getAction()) + .collect(Collectors.toList()); + } + + public enum ImportAction { + APPEND_OVERWRITE("append-overwrite-observation"), + NEW_OBSERVATION("new-observation"), + APPEND_ENVIRONMENT("append-environment"); + + private String urlFragment; + + ImportAction(String urlFragment) { + this.urlFragment = urlFragment; + } + + public String getUrlFragment() { + return urlFragment; + } + + public boolean isEqual(String value) { + return Optional.ofNullable(urlFragment.equals(value)).orElse(false); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java new file mode 100644 index 000000000..ca5e932ef --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java @@ -0,0 +1,53 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; + +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; + +import javax.inject.Singleton; +import java.util.Optional; + +@Singleton +public class AppendOverwriteObservationWorkflow implements ImportWorkflow { + private final ExperimentImportWorkflow.ImportAction action; + + public AppendOverwriteObservationWorkflow(){ + this.action = ExperimentImportWorkflow.ImportAction.APPEND_OVERWRITE; + } + + @Override + public Optional process(ImportServiceContext context) { + // Workflow processing the context + Action workflow = new Action(action.getUrlFragment(), this.getOrder()); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .action(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless appending or overwriting observation data + if (context != null && !action.isEqual(context.getAction())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; + } + + @Override + public int getOrder() { + return 2; + } + + public ExperimentImportWorkflow.ImportAction getAction() { + return this.action; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java new file mode 100644 index 000000000..c29fe9fd5 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java @@ -0,0 +1,54 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; + +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; + +import javax.inject.Singleton; +import java.util.Optional; + +@Singleton +public class NewExperimentWorkflow implements ImportWorkflow { + private final ExperimentImportWorkflow.ImportAction action; + + public NewExperimentWorkflow(){ + this.action = ExperimentImportWorkflow.ImportAction.NEW_OBSERVATION; + } + + @Override + public Optional process(ImportServiceContext context) { + // Workflow processing the context + Action workflow = new Action(action.getUrlFragment(), this.getOrder()); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .action(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless creating a new experiment + if (context != null && !action.isEqual(context.getAction())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; + } + + @Override + public int getOrder() { + return 1; + } + + public ExperimentImportWorkflow.ImportAction getAction() { + return this.action; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java new file mode 100644 index 000000000..57ee9790e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java @@ -0,0 +1,53 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; + +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; + +import javax.inject.Singleton; +import java.util.Optional; + +@Singleton +public class NewEnvironmentWorkflow implements ImportWorkflow { + private final ExperimentImportWorkflow.ImportAction action; + + public NewEnvironmentWorkflow(){ + this.action = ExperimentImportWorkflow.ImportAction.APPEND_ENVIRONMENT; + } + + @Override + public Optional process(ImportServiceContext context) { + // Workflow processing the context + Action workflow = new Action(action.getUrlFragment(), this.getOrder()); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .action(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless appending a new environment + if (context != null && !action.isEqual(context.getAction())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; + } + + @Override + public int getOrder() { + return 3; + } + + public ExperimentImportWorkflow.ImportAction getAction() { + return this.action; + } +} From f726611cef1f5f3b3d82f76fd9b0c3ad5e8c8a01 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 22 May 2024 18:52:39 -0400 Subject: [PATCH 067/203] create FileImportService method for GET workflows --- .../importer/controllers/ImportController.java | 13 ++++++++++--- .../importer/model/imports/BrAPIImportService.java | 4 ++++ .../ExperimentImportService.java | 5 +++++ .../brapps/importer/services/FileImportService.java | 9 +++++++-- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index 5c9ac0ca1..e9e9612e1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -37,6 +37,7 @@ import org.breedinginsight.api.model.v1.response.metadata.StatusCode; import org.breedinginsight.api.v1.controller.metadata.AddMetadata; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; +import org.breedinginsight.brapps.importer.model.workflow.Action; import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; import org.breedinginsight.brapps.importer.services.ImportConfigManager; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; @@ -214,16 +215,22 @@ public HttpResponse>> getSystemMappings(@Nu @Produces(MediaType.APPLICATION_JSON) @AddMetadata @Secured(SecurityRule.IS_ANONYMOUS) - public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { + public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { - List workflows = fileImportService.getWorkflowsForSystemMapping(mappingId); + List workflows = null; + try { + workflows = fileImportService.getWorkflowsForSystemMapping(mappingId); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } List metadataStatus = new ArrayList<>(); metadataStatus.add(new Status(StatusCode.INFO, "Successful Query")); Pagination pagination = new Pagination(workflows.size(), workflows.size(), 1, 0); Metadata metadata = new Metadata(pagination, metadataStatus); - Response> response = new Response(metadata, new DataResponse<>(workflows)); + Response> response = new Response(metadata, new DataResponse<>(workflows)); return HttpResponse.ok(response); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index d06454c30..997155d93 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -18,10 +18,14 @@ package org.breedinginsight.brapps.importer.model.imports; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.Action; + +import java.util.List; public interface BrAPIImportService { String getImportTypeId(); BrAPIImport getImportClass(); + List getWorkflows(); default String getInvalidIntegerMsg(String columnName) { return String.format("Column name \"%s\" must be integer type, but non-integer type provided.", columnName); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index de7d54961..760464d93 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -23,6 +23,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.Action; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; @@ -72,6 +73,10 @@ public String getImportTypeId() { public String getMissingColumnMsg(String columnName) { return "Column heading does not match template or ontology"; } + @Override + public List getWorkflows() { + return workflow.getWorkflows(); + } @Override public ImportPreviewResponse process(ImportServiceContext context) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 59438c696..7d9ebc0c6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -43,6 +43,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; +import org.breedinginsight.brapps.importer.model.workflow.Action; import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.workflow.WorkflowFactory; @@ -584,8 +585,12 @@ public List getSystemMappingByName(String name) { return importMappings; } - public List getWorkflowsForSystemMapping(UUID mappingId) { - return importMappingWorkflowDAO.getWorkflowsByImportMappingId(mappingId); + public List getWorkflowsForSystemMapping(UUID mappingId) throws DoesNotExistException { + ImportMapping mappingConfig = importMappingDAO.getMapping(mappingId) + .orElseThrow(() -> new DoesNotExistException("Cannot find mapping config associated with upload.")); + BrAPIImportService importService = configManager.getImportServiceById(mappingConfig.getImportTypeId()) + .orElseThrow(() -> new DoesNotExistException("Config with that id does not exist")); + return importService.getWorkflows(); } From f6952dd78f45955706e5adbe25eed24ab7b641e2 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 22 May 2024 19:22:18 -0400 Subject: [PATCH 068/203] delete unused classes --- .../controllers/UploadController.java | 12 +-- .../daos/ImportMappingWorkflowDAO.java | 68 --------------- .../importer/services/FileImportService.java | 24 ++---- .../services/workflow/WorkflowFactory.java | 83 ------------------- 4 files changed, 11 insertions(+), 176 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java index 053f1dc3c..5ee6eb062 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java @@ -158,15 +158,15 @@ public HttpResponse> previewData(@PathVariable UUID pro } } - @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/preview") + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflow}/data/{uploadId}/preview") @Produces(MediaType.APPLICATION_JSON) @AddMetadata @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) public HttpResponse> previewData(@PathVariable UUID programId, @PathVariable UUID mappingId, - @PathVariable UUID workflowId, @PathVariable UUID uploadId) { + @PathVariable String workflow, @PathVariable UUID uploadId) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, null, false); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflow, actingUser, null, false); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { @@ -184,16 +184,16 @@ public HttpResponse> previewData(@PathVariable UUID pro } } - @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/commit") + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflow}/data/{uploadId}/commit") @Produces(MediaType.APPLICATION_JSON) @AddMetadata @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) public HttpResponse> commitData(@PathVariable UUID programId, @PathVariable UUID mappingId, - @PathVariable UUID workflowId, @PathVariable UUID uploadId, + @PathVariable String workflow, @PathVariable UUID uploadId, @Body @Nullable Map userInput) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, userInput, true); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflow, actingUser, userInput, true); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java deleted file mode 100644 index 1b60bda45..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.daos; - -import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; -import org.breedinginsight.dao.db.tables.daos.ImporterMappingWorkflowDao; -import org.jooq.Configuration; -import org.jooq.DSLContext; - -import javax.inject.Inject; -import java.util.List; -import java.util.Optional; -import java.util.UUID; - -import static org.breedinginsight.dao.db.Tables.*; - -public class ImportMappingWorkflowDAO extends ImporterMappingWorkflowDao { - - private DSLContext dsl; - - @Inject - public ImportMappingWorkflowDAO(Configuration config, DSLContext dsl) { - super(config); - this.dsl = dsl; - } - - /** - * Retrieves a list of ImportMappingWorkflow objects associated with the given mappingId. They are ordered by - * position for proper ordering on the front end. - * - * @param mappingId The UUID of the mapping to retrieve the workflows for. - * @return A list of ImportMappingWorkflow objects. - */ - public List getWorkflowsByImportMappingId(UUID mappingId) { - return dsl.select() - .from(IMPORTER_MAPPING_WORKFLOW) - .where(IMPORTER_MAPPING_WORKFLOW.MAPPING_ID.eq(mappingId)) - .orderBy(IMPORTER_MAPPING_WORKFLOW.POSITION.asc()) - .fetch(ImportMappingWorkflow::parseSQLRecord); - } - - /** - * Retrieves a workflow by its ID. - * - * @param workflowId The ID of the workflow to retrieve. - * @return An Optional containing the ImportMappingWorkflow if found, otherwise an empty Optional. - */ - public Optional getWorkflowById(UUID workflowId) { - return Optional.ofNullable(fetchOneById(workflowId)) - .map(ImportMappingWorkflow::new); - } - -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 7d9ebc0c6..ffaa68614 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.micronaut.http.HttpStatus; -import io.micronaut.http.annotation.PathVariable; import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.multipart.CompletedFileUpload; import io.micronaut.http.server.exceptions.InternalServerException; @@ -33,7 +32,6 @@ import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.daos.ImportMappingProgramDAO; -import org.breedinginsight.brapps.importer.daos.ImportMappingWorkflowDAO; import org.breedinginsight.brapps.importer.model.ImportProgress; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; @@ -44,12 +42,8 @@ import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; -import org.breedinginsight.brapps.importer.services.workflow.WorkflowFactory; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingProgramEntity; -import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import org.breedinginsight.services.ProgramService; @@ -88,14 +82,12 @@ public class FileImportService { private final ImportDAO importDAO; private final DSLContext dsl; private final ImportMappingProgramDAO importMappingProgramDAO; - private final ImportMappingWorkflowDAO importMappingWorkflowDAO; - private final WorkflowFactory workflowFactory; @Inject FileImportService(ProgramUserService programUserService, ProgramService programService, MimeTypeParser mimeTypeParser, ImportMappingDAO importMappingDAO, ObjectMapper objectMapper, MappingManager mappingManager, ImportConfigManager configManager, ImportDAO importDAO, DSLContext dsl, ImportMappingProgramDAO importMappingProgramDAO, - ImportMappingWorkflowDAO importMappingWorkflowDAO, WorkflowFactory workflowFactory, UserService userService) { + UserService userService) { this.programUserService = programUserService; this.programService = programService; this.mimeTypeParser = mimeTypeParser; @@ -107,8 +99,6 @@ public class FileImportService { this.dsl = dsl; this.importMappingProgramDAO = importMappingProgramDAO; this.userService = userService; - this.importMappingWorkflowDAO = importMappingWorkflowDAO; - this.workflowFactory = workflowFactory; } public List getAllImportTypeConfigs() { @@ -334,7 +324,7 @@ public ImportResponse uploadData(UUID programId, UUID mappingId, AuthenticatedUs return response; } - public ImportResponse updateUpload(UUID programId, UUID uploadId, UUID workflowId, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws + public ImportResponse updateUpload(UUID programId, UUID uploadId, String workflow, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws DoesNotExistException, UnprocessableEntityException, AuthorizationException { Program program = validateRequest(programId, actingUser); @@ -384,7 +374,7 @@ public ImportResponse updateUpload(UUID programId, UUID uploadId, UUID workflowI } else { brAPIImportList = mappingManager.map(mappingConfig, data); } - processFile(workflowId, brAPIImportList, data, program, upload, user, commit, importService, actingUser); + processFile(workflow, brAPIImportList, data, program, upload, user, commit, importService, actingUser); } catch (UnprocessableEntityException e) { log.error(e.getMessage(), e); ImportProgress progress = upload.getProgress(); @@ -430,18 +420,14 @@ public ImportUpload setDynamicColumns(ImportUpload newUpload, Table data, Import return newUpload; } - private void processFile(UUID workflowId, List finalBrAPIImportList, Table data, Program program, + private void processFile(String workflow, List finalBrAPIImportList, Table data, Program program, ImportUpload upload, User user, Boolean commit, BrAPIImportService importService, AuthenticatedUser actingUser) { // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { - - final Workflow[] workflow = {null}; - Optional.ofNullable(workflowId).ifPresent(id -> workflow[0] = workflowFactory.getWorkflow(id).orElse(null)); - ImportServiceContext context = ImportServiceContext.builder() - .workflow(workflow[0]) + .action(workflow) .brAPIImports(finalBrAPIImportList) .data(data) .program(program) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java deleted file mode 100644 index 17d6fe64e..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.services.workflow; - -import io.micronaut.context.ApplicationContext; -import io.micronaut.context.annotation.Factory; -import io.micronaut.context.exceptions.NoSuchBeanException; -import io.micronaut.inject.qualifiers.Qualifiers; -import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.daos.ImportMappingWorkflowDAO; -import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; - -import javax.inject.Inject; -import java.util.Optional; -import java.util.UUID; - -@Factory -@Slf4j -public class WorkflowFactory { - - private final ImportMappingWorkflowDAO importMappingWorkflowDAO; - private final ApplicationContext applicationContext; - - @Inject - public WorkflowFactory(ImportMappingWorkflowDAO importMappingWorkflowDAO, - ApplicationContext applicationContext) { - this.importMappingWorkflowDAO = importMappingWorkflowDAO; - this.applicationContext = applicationContext; - } - - /** - * Produces the appropriate workflow instance based on the import context - * - * @param context the import context - * @return an Optional containing the workflow if id is not null, otherwise an empty Optional - * - * @throws IllegalStateException - * @throws NoSuchBeanException - */ - public Optional getWorkflow(UUID workflowId) { - - if (workflowId != null) { - // construct workflow from db record - Optional workflowOptional = importMappingWorkflowDAO.getWorkflowById(workflowId); - - ImportMappingWorkflow importMappingworkflow = workflowOptional.orElseThrow(() -> { - String msg = "Must have record in db for workflowId"; - log.error(msg); - return new IllegalStateException(msg); - }); - - // newer versions of micronaut have fancier ways to do this using annotations with provider but as - // far as I can tell it's not available in 2.5 - Workflow workflow; - try { - workflow = applicationContext.getBean(Workflow.class, Qualifiers.byName(importMappingworkflow.getBean())); - } catch (NoSuchBeanException e) { - log.error("Could not find workflow class implementation for bean: " + importMappingworkflow.getBean()); - throw e; - } - - return Optional.of(workflow); - } - - return Optional.empty(); - } -} From 5e31aa6f363f7b1767b49fe59b79c793cd10a373 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 22 May 2024 20:33:43 -0400 Subject: [PATCH 069/203] clean up and renaming --- .../controllers/ImportController.java | 9 +-- .../model/imports/BrAPIImportService.java | 4 +- .../model/imports/ImportServiceContext.java | 5 +- .../ExperimentImportService.java | 22 ++--- .../importer/model/workflow/Action.java | 29 ------- .../model/workflow/ImportWorkflow.java | 35 ++++++-- .../model/workflow/ImportWorkflowResult.java | 2 +- .../importer/model/workflow/Workflow.java | 37 ++------- .../importer/services/FileImportService.java | 6 +- ...tWorkflow.java => ExperimentWorkflow.java} | 26 +++--- .../AppendOverwriteObservationWorkflow.java | 53 ------------ .../AppendOverwritePhenotypesWorkflow.java | 81 ++++++++++--------- .../workflow/CreateNewExperimentWorkflow.java | 79 +++++++++--------- .../workflow/NewExperimentWorkflow.java | 54 ------------- .../CreateNewEnvironmentWorkflow.java | 81 ++++++++++--------- .../workflow/NewEnvironmentWorkflow.java | 53 ------------ .../V1.22.0__add_experiment_workflows.sql | 51 ------------ 17 files changed, 195 insertions(+), 432 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ExperimentImportWorkflow.java => ExperimentWorkflow.java} (61%) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java delete mode 100644 src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index e9e9612e1..1a16dadd7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -37,8 +37,7 @@ import org.breedinginsight.api.model.v1.response.metadata.StatusCode; import org.breedinginsight.api.v1.controller.metadata.AddMetadata; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; -import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.ImportConfigManager; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; import org.breedinginsight.brapps.importer.services.FileImportService; @@ -215,9 +214,9 @@ public HttpResponse>> getSystemMappings(@Nu @Produces(MediaType.APPLICATION_JSON) @AddMetadata @Secured(SecurityRule.IS_ANONYMOUS) - public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { + public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { - List workflows = null; + List workflows = null; try { workflows = fileImportService.getWorkflowsForSystemMapping(mappingId); } catch (DoesNotExistException e) { @@ -230,7 +229,7 @@ public HttpResponse>> getWorkflowsForSystemMapping Pagination pagination = new Pagination(workflows.size(), workflows.size(), 1, 0); Metadata metadata = new Metadata(pagination, metadataStatus); - Response> response = new Response(metadata, new DataResponse<>(workflows)); + Response> response = new Response(metadata, new DataResponse<>(workflows)); return HttpResponse.ok(response); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index 997155d93..3f25d99e6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -18,14 +18,14 @@ package org.breedinginsight.brapps.importer.model.imports; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import java.util.List; public interface BrAPIImportService { String getImportTypeId(); BrAPIImport getImportClass(); - List getWorkflows(); + List getWorkflows(); default String getInvalidIntegerMsg(String columnName) { return String.format("Column name \"%s\" must be integer type, but non-integer type provided.", columnName); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java index c5a479e90..4d052cd33 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -19,10 +19,10 @@ import lombok.*; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import tech.tablesaw.api.Table; + import java.util.List; @Getter @@ -32,8 +32,7 @@ @AllArgsConstructor @NoArgsConstructor public class ImportServiceContext { - private String action; - private Workflow workflow; + private String workflow; private List brAPIImports; private Table data; private Program program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 760464d93..df60e9782 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -18,26 +18,18 @@ package org.breedinginsight.brapps.importer.model.imports.experimentObservation; import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; -import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import tech.tablesaw.api.Table; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; import java.util.List; -import java.util.Optional; @Singleton @Slf4j @@ -47,12 +39,12 @@ public class ExperimentImportService implements BrAPIImportService { private final Provider experimentProcessorProvider; private final Provider processorManagerProvider; - private final ExperimentImportWorkflow workflow; + private final ExperimentWorkflow workflow; @Inject public ExperimentImportService(Provider experimentProcessorProvider, Provider processorManagerProvider, - ExperimentImportWorkflow workflow) + ExperimentWorkflow workflow) { this.experimentProcessorProvider = experimentProcessorProvider; this.processorManagerProvider = processorManagerProvider; @@ -74,7 +66,7 @@ public String getMissingColumnMsg(String columnName) { return "Column heading does not match template or ontology"; } @Override - public List getWorkflows() { + public List getWorkflows() { return workflow.getWorkflows(); } @@ -82,8 +74,8 @@ public List getWorkflows() { public ImportPreviewResponse process(ImportServiceContext context) throws Exception { - if (!context.getAction().isEmpty()) { - log.info("Workflow: " + context.getAction()); + if (!context.getWorkflow().isEmpty()) { + log.info("Workflow: " + context.getWorkflow()); } return workflow.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java deleted file mode 100644 index 65c3d0cbb..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java +++ /dev/null @@ -1,29 +0,0 @@ -///* -// * See the NOTICE file distributed with this work for additional information -// * regarding copyright ownership. -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ - -package org.breedinginsight.brapps.importer.model.workflow; - -import lombok.*; - -@Getter -@Setter -@ToString -@AllArgsConstructor -public class Action { - private String name; - private int order; -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java index c5dfc1957..b26a886bd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java @@ -1,12 +1,31 @@ -package org.breedinginsight.brapps.importer.model.workflow; +///* +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ -import io.micronaut.core.order.Ordered; -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +package org.breedinginsight.brapps.importer.model.workflow; -import java.util.Optional; +import lombok.*; -@FunctionalInterface -public interface ImportWorkflow extends Ordered { - Optional process(ImportServiceContext context); +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +public class ImportWorkflow { + private String urlFragment; + private String displayName; + private int order; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java index 2e61a6d46..9304c1ce2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java @@ -28,6 +28,6 @@ @ToString @AllArgsConstructor public class ImportWorkflowResult { - private Action action; + private ImportWorkflow workflow; private Optional importPreviewResponse; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java index 9ea0e7bf9..8bc19120e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -1,36 +1,11 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.breedinginsight.brapps.importer.model.workflow; -public interface Workflow { +import io.micronaut.core.order.Ordered; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; - /** - * Processes the given import context and returns the processed data. - * - * @param context the import context containing the necessary data for processing - * @return the processed data - */ - ProcessedData process(ImportContext context); +import java.util.Optional; - /** - * Retrieves the name of the Workflow for logging display purposes. - * - * @return the name of the Workflow - */ - String getName(); +@FunctionalInterface +public interface Workflow extends Ordered { + Optional process(ImportServiceContext context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index ffaa68614..6a1914ab9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -41,7 +41,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; -import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingProgramEntity; import org.breedinginsight.model.Program; @@ -427,7 +427,7 @@ private void processFile(String workflow, List finalBrAPIImportList CompletableFuture.supplyAsync(() -> { try { ImportServiceContext context = ImportServiceContext.builder() - .action(workflow) + .workflow(workflow) .brAPIImports(finalBrAPIImportList) .data(data) .program(program) @@ -571,7 +571,7 @@ public List getSystemMappingByName(String name) { return importMappings; } - public List getWorkflowsForSystemMapping(UUID mappingId) throws DoesNotExistException { + public List getWorkflowsForSystemMapping(UUID mappingId) throws DoesNotExistException { ImportMapping mappingConfig = importMappingDAO.getMapping(mappingId) .orElseThrow(() -> new DoesNotExistException("Cannot find mapping config associated with upload.")); BrAPIImportService importService = configManager.getImportServiceById(mappingConfig.getImportTypeId()) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java similarity index 61% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java index e8a7d45da..04bbc4fc1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java @@ -2,8 +2,8 @@ import io.micronaut.context.annotation.Primary; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.workflow.Action; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import javax.inject.Singleton; @@ -13,10 +13,10 @@ @Primary @Singleton -public class ExperimentImportWorkflow implements ImportWorkflow { - private final List workflows; +public class ExperimentWorkflow implements Workflow { + private final List workflows; - public ExperimentImportWorkflow(List workflows) { + public ExperimentWorkflow(List workflows) { this.workflows = workflows; } @@ -28,29 +28,33 @@ public Optional process(ImportServiceContext context) { .map(Optional::get) .findFirst(); } - public List getWorkflows() { + public List getWorkflows() { return workflows.stream() .map(workflow->workflow.process(null)) .filter(Optional::isPresent) .map(Optional::get) - .map(result->result.getAction()) + .map(result->result.getWorkflow()) .collect(Collectors.toList()); } - public enum ImportAction { - APPEND_OVERWRITE("append-overwrite-observation"), - NEW_OBSERVATION("new-observation"), - APPEND_ENVIRONMENT("append-environment"); + public enum Workflow { + NEW_OBSERVATION("new-experiment","Create new experiment"), + APPEND_OVERWRITE("append-dataset", "Append experimental dataset"), + APPEND_ENVIRONMENT("append-environment", "Create new experimental environment"); private String urlFragment; + private String displayName; + + Workflow(String urlFragment, String displayName) { - ImportAction(String urlFragment) { this.urlFragment = urlFragment; + this.displayName = displayName; } public String getUrlFragment() { return urlFragment; } + public String getDisplayName() { return displayName; } public boolean isEqual(String value) { return Optional.ofNullable(urlFragment.equals(value)).orElse(false); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java deleted file mode 100644 index ca5e932ef..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; - -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; - -import javax.inject.Singleton; -import java.util.Optional; - -@Singleton -public class AppendOverwriteObservationWorkflow implements ImportWorkflow { - private final ExperimentImportWorkflow.ImportAction action; - - public AppendOverwriteObservationWorkflow(){ - this.action = ExperimentImportWorkflow.ImportAction.APPEND_OVERWRITE; - } - - @Override - public Optional process(ImportServiceContext context) { - // Workflow processing the context - Action workflow = new Action(action.getUrlFragment(), this.getOrder()); - - // No-preview result - Optional result = Optional.of(ImportWorkflowResult.builder() - .action(workflow) - .importPreviewResponse(Optional.empty()) - .build()); - - // Skip this workflow unless appending or overwriting observation data - if (context != null && !action.isEqual(context.getAction())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return result; - } - - // Start processing the import... - return result; - } - - @Override - public int getOrder() { - return 2; - } - - public ExperimentImportWorkflow.ImportAction getAction() { - return this.action; - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index 21df19be6..44fb575bf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -1,50 +1,55 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; -import io.micronaut.context.annotation.Prototype; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import lombok.Getter; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; -import javax.inject.Named; - -/** - * This class represents a workflow for appending and overwriting phenotypes. The bean name must match the appropriate - * bean column value in the import_mapping_workflow db table - */ +import javax.inject.Singleton; +import java.util.Optional; -@Prototype -@Named("AppendOverwritePhenotypesWorkflow") +@Getter +@Singleton public class AppendOverwritePhenotypesWorkflow implements Workflow { + private final ExperimentWorkflow.Workflow workflow; + + public AppendOverwritePhenotypesWorkflow(){ + this.workflow = ExperimentWorkflow.Workflow.APPEND_OVERWRITE; + } + @Override - public ProcessedData process(ImportContext context) { - // TODO - return null; + public Optional process(ImportServiceContext context) { + // Workflow processing the context + ImportWorkflow workflow = ImportWorkflow.builder() + .urlFragment(getWorkflow().getUrlFragment()) + .displayName(getWorkflow().getDisplayName()) + .build(); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .workflow(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless appending or overwriting observation data + if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; } - /** - * Retrieves the name of the workflow. This is used for logging display purposes. - * - * @return the name of the workflow - */ @Override - public String getName() { - return "AppendOverwritePhenotypesWorkflow"; + public int getOrder() { + return 2; } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 6f6ab661a..3c593ac74 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -1,50 +1,55 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; -import io.micronaut.context.annotation.Prototype; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import lombok.Getter; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; -import javax.inject.Named; +import javax.inject.Singleton; +import java.util.Optional; -/** - * This class represents a workflow for creating a new experiment. The bean name must match the appropriate bean column - * value in the import_mapping_workflow db table - */ -@Prototype -@Named("CreateNewExperimentWorkflow") +@Getter +@Singleton public class CreateNewExperimentWorkflow implements Workflow { + private final ExperimentWorkflow.Workflow workflow; + + public CreateNewExperimentWorkflow(){ + this.workflow = ExperimentWorkflow.Workflow.NEW_OBSERVATION; + } @Override - public ProcessedData process(ImportContext context) { - // TODO - return null; + public Optional process(ImportServiceContext context) { + // Workflow processing the context + ImportWorkflow workflow = ImportWorkflow.builder() + .urlFragment(getWorkflow().getUrlFragment()) + .displayName(getWorkflow().getDisplayName()) + .build(); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .workflow(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless creating a new experiment + if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; } - /** - * Retrieves the name of the workflow. This is used for logging display purposes. - * - * @return the name of the workflow - */ @Override - public String getName() { - return "CreateNewExperimentWorkflow"; + public int getOrder() { + return 1; } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java deleted file mode 100644 index c29fe9fd5..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; - -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; - -import javax.inject.Singleton; -import java.util.Optional; - -@Singleton -public class NewExperimentWorkflow implements ImportWorkflow { - private final ExperimentImportWorkflow.ImportAction action; - - public NewExperimentWorkflow(){ - this.action = ExperimentImportWorkflow.ImportAction.NEW_OBSERVATION; - } - - @Override - public Optional process(ImportServiceContext context) { - // Workflow processing the context - Action workflow = new Action(action.getUrlFragment(), this.getOrder()); - - // No-preview result - Optional result = Optional.of(ImportWorkflowResult.builder() - .action(workflow) - .importPreviewResponse(Optional.empty()) - .build()); - - // Skip this workflow unless creating a new experiment - if (context != null && !action.isEqual(context.getAction())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return result; - } - - // Start processing the import... - return result; - } - - @Override - public int getOrder() { - return 1; - } - - public ExperimentImportWorkflow.ImportAction getAction() { - return this.action; - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java index 5776ab59b..fe0b90c1f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java @@ -1,50 +1,55 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; -import io.micronaut.context.annotation.Prototype; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import lombok.Getter; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; -import javax.inject.Named; - -/** - * This class represents a workflow for adding new environments to an existing experiment. The bean name must match - * the appropriate bean column value in the import_mapping_workflow db table - */ +import javax.inject.Singleton; +import java.util.Optional; -@Prototype -@Named("CreateNewEnvironmentWorkflow") +@Getter +@Singleton public class CreateNewEnvironmentWorkflow implements Workflow { + private final ExperimentWorkflow.Workflow workflow; + + public CreateNewEnvironmentWorkflow(){ + this.workflow = ExperimentWorkflow.Workflow.APPEND_ENVIRONMENT; + } + @Override - public ProcessedData process(ImportContext context) { - // TODO - return null; + public Optional process(ImportServiceContext context) { + // Workflow processing the context + ImportWorkflow workflow = ImportWorkflow.builder() + .urlFragment(getWorkflow().getUrlFragment()) + .displayName(getWorkflow().getDisplayName()) + .build(); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .workflow(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless appending a new environment + if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; } - /** - * Retrieves the name of the workflow. This is used for logging display purposes. - * - * @return the name of the workflow - */ @Override - public String getName() { - return "CreateNewEnvironmentWorkflow"; + public int getOrder() { + return 3; } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java deleted file mode 100644 index 57ee9790e..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; - -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; - -import javax.inject.Singleton; -import java.util.Optional; - -@Singleton -public class NewEnvironmentWorkflow implements ImportWorkflow { - private final ExperimentImportWorkflow.ImportAction action; - - public NewEnvironmentWorkflow(){ - this.action = ExperimentImportWorkflow.ImportAction.APPEND_ENVIRONMENT; - } - - @Override - public Optional process(ImportServiceContext context) { - // Workflow processing the context - Action workflow = new Action(action.getUrlFragment(), this.getOrder()); - - // No-preview result - Optional result = Optional.of(ImportWorkflowResult.builder() - .action(workflow) - .importPreviewResponse(Optional.empty()) - .build()); - - // Skip this workflow unless appending a new environment - if (context != null && !action.isEqual(context.getAction())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return result; - } - - // Start processing the import... - return result; - } - - @Override - public int getOrder() { - return 3; - } - - public ExperimentImportWorkflow.ImportAction getAction() { - return this.action; - } -} diff --git a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql deleted file mode 100644 index 2c9d4d547..000000000 --- a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql +++ /dev/null @@ -1,51 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - Table maps workflows to import mappings and provides required configuration options - - mapping_id - link to importer_mapping that this provides a workflow for - name - name that will be displayed on front end - bean - must match @Named("") annotation on Workflow class - position - for ordering records explicitly, wanted for front end default option and order - */ -CREATE TABLE importer_mapping_workflow -( - like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, - mapping_id UUID NOT NULL, - name TEXT NOT NULL, - bean TEXT NOT NULL, - position INTEGER NOT NULL -); - -ALTER TABLE importer_mapping_workflow - ADD FOREIGN KEY (mapping_id) REFERENCES importer_mapping (id); - -DO -$$ -DECLARE - exp_mapping_id UUID; -BEGIN - exp_mapping_id := (SELECT id FROM importer_mapping WHERE name = 'ExperimentsTemplateMap'); - -INSERT INTO public.importer_mapping_workflow (mapping_id, name, bean, position) -VALUES - (exp_mapping_id, 'Create new experiment', 'CreateNewExperimentWorkflow', 0), - (exp_mapping_id, 'Append experimental dataset', 'AppendOverwritePhenotypesWorkflow', 1), - (exp_mapping_id, 'Create new experimental environment', 'CreateNewEnvironmentWorkflow', 2); -END -$$; \ No newline at end of file From 708793dae1c11e78ece376f1a5fdd3813fcc821d Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 22 May 2024 20:38:09 -0400 Subject: [PATCH 070/203] add overrides to import services --- .../model/imports/germplasm/GermplasmImportService.java | 6 ++++++ .../model/imports/sample/SampleSubmissionImportService.java | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index ce52f483f..3c3a65b4d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -23,6 +23,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.GermplasmProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; @@ -57,6 +58,11 @@ public GermplasmImport getImportClass() { return new GermplasmImport(); } + @Override + public List getWorkflows() { + return null; + } + @Override public String getImportTypeId() { return IMPORT_TYPE_ID; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java index 2c8e8b7b5..55848b0a1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java @@ -23,6 +23,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.brapps.importer.services.processors.SampleSubmissionProcessor; @@ -59,6 +60,11 @@ public BrAPIImport getImportClass() { return new SampleSubmissionImport(); } + @Override + public List getWorkflows() { + return null; + } + @Override public ImportPreviewResponse process(ImportServiceContext context) throws Exception { List processors = List.of(sampleProcessorProvider.get()); From 68bdacb282b335af1874db9ede5ad421de4806ae Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 23 May 2024 06:33:16 -0400 Subject: [PATCH 071/203] create ExperimentWorkflow interface that extends Workflow interface --- .../ExperimentImportService.java | 12 ++++++------ .../importer/model/workflow/ExperimentWorkflow.java | 6 ++++++ .../brapps/importer/model/workflow/Workflow.java | 11 ----------- ...orkflow.java => ExperimentWorkflowNavigator.java} | 8 ++++---- .../workflow/AppendOverwritePhenotypesWorkflow.java | 10 +++++----- .../create/workflow/CreateNewExperimentWorkflow.java | 10 +++++----- .../workflow/CreateNewEnvironmentWorkflow.java | 10 +++++----- 7 files changed, 31 insertions(+), 36 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ExperimentWorkflow.java => ExperimentWorkflowNavigator.java} (85%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index df60e9782..c6d191bf0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -24,7 +24,7 @@ import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import javax.inject.Inject; import javax.inject.Provider; @@ -39,16 +39,16 @@ public class ExperimentImportService implements BrAPIImportService { private final Provider experimentProcessorProvider; private final Provider processorManagerProvider; - private final ExperimentWorkflow workflow; + private final ExperimentWorkflowNavigator workflowNavigator; @Inject public ExperimentImportService(Provider experimentProcessorProvider, Provider processorManagerProvider, - ExperimentWorkflow workflow) + ExperimentWorkflowNavigator workflowNavigator) { this.experimentProcessorProvider = experimentProcessorProvider; this.processorManagerProvider = processorManagerProvider; - this.workflow = workflow; + this.workflowNavigator = workflowNavigator; } @Override @@ -67,7 +67,7 @@ public String getMissingColumnMsg(String columnName) { } @Override public List getWorkflows() { - return workflow.getWorkflows(); + return workflowNavigator.getWorkflows(); } @Override @@ -78,7 +78,7 @@ public ImportPreviewResponse process(ImportServiceContext context) log.info("Workflow: " + context.getWorkflow()); } - return workflow.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); + return workflowNavigator.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java new file mode 100644 index 000000000..c34794bab --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java @@ -0,0 +1,6 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +@FunctionalInterface +public interface ExperimentWorkflow extends Workflow { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java deleted file mode 100644 index 8bc19120e..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.breedinginsight.brapps.importer.model.workflow; - -import io.micronaut.core.order.Ordered; -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; - -import java.util.Optional; - -@FunctionalInterface -public interface Workflow extends Ordered { - Optional process(ImportServiceContext context); -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java similarity index 85% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index 04bbc4fc1..bcf12d543 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -3,7 +3,7 @@ import io.micronaut.context.annotation.Primary; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import javax.inject.Singleton; @@ -13,10 +13,10 @@ @Primary @Singleton -public class ExperimentWorkflow implements Workflow { - private final List workflows; +public class ExperimentWorkflowNavigator implements ExperimentWorkflow { + private final List workflows; - public ExperimentWorkflow(List workflows) { + public ExperimentWorkflowNavigator(List workflows) { this.workflows = workflows; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index 44fb575bf..e64a2f88a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -4,19 +4,19 @@ import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import javax.inject.Singleton; import java.util.Optional; @Getter @Singleton -public class AppendOverwritePhenotypesWorkflow implements Workflow { - private final ExperimentWorkflow.Workflow workflow; +public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow { + private final ExperimentWorkflowNavigator.Workflow workflow; public AppendOverwritePhenotypesWorkflow(){ - this.workflow = ExperimentWorkflow.Workflow.APPEND_OVERWRITE; + this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE; } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 3c593ac74..022727c5a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -4,19 +4,19 @@ import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import javax.inject.Singleton; import java.util.Optional; @Getter @Singleton -public class CreateNewExperimentWorkflow implements Workflow { - private final ExperimentWorkflow.Workflow workflow; +public class CreateNewExperimentWorkflow implements ExperimentWorkflow { + private final ExperimentWorkflowNavigator.Workflow workflow; public CreateNewExperimentWorkflow(){ - this.workflow = ExperimentWorkflow.Workflow.NEW_OBSERVATION; + this.workflow = ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION; } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java index fe0b90c1f..a5695e7f8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java @@ -4,19 +4,19 @@ import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import javax.inject.Singleton; import java.util.Optional; @Getter @Singleton -public class CreateNewEnvironmentWorkflow implements Workflow { - private final ExperimentWorkflow.Workflow workflow; +public class CreateNewEnvironmentWorkflow implements ExperimentWorkflow { + private final ExperimentWorkflowNavigator.Workflow workflow; public CreateNewEnvironmentWorkflow(){ - this.workflow = ExperimentWorkflow.Workflow.APPEND_ENVIRONMENT; + this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_ENVIRONMENT; } @Override From 0e022f12675712ad57574d4b7a27e2e5cac6700f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 23 May 2024 06:48:39 -0400 Subject: [PATCH 072/203] create interfaces for germplasm and sample workflows --- .../model/workflow/ImportMappingWorkflow.java | 56 ------------------- 1 file changed, 56 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java deleted file mode 100644 index 8dc5800bc..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.model.workflow; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; -import lombok.experimental.Accessors; -import lombok.experimental.SuperBuilder; - -import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; -import org.jooq.Record; - -import static org.breedinginsight.dao.db.tables.ImporterMappingWorkflowTable.IMPORTER_MAPPING_WORKFLOW; - -@Getter -@Setter -@Accessors(chain=true) -@ToString -@NoArgsConstructor -@SuperBuilder() -public class ImportMappingWorkflow extends ImporterMappingWorkflowEntity { - - - public ImportMappingWorkflow(ImporterMappingWorkflowEntity importMappingWorkflowEntity) { - this.setId(importMappingWorkflowEntity.getId()); - this.setName(importMappingWorkflowEntity.getName()); - this.setBean(importMappingWorkflowEntity.getBean()); - this.setPosition(importMappingWorkflowEntity.getPosition()); - } - public static ImportMappingWorkflow parseSQLRecord(Record record) { - - return ImportMappingWorkflow.builder() - .id(record.getValue(IMPORTER_MAPPING_WORKFLOW.ID)) - .name(record.getValue(IMPORTER_MAPPING_WORKFLOW.NAME)) - .bean(record.getValue(IMPORTER_MAPPING_WORKFLOW.BEAN)) - .position(record.getValue(IMPORTER_MAPPING_WORKFLOW.POSITION)) - .build(); - } -} From f2a1034abf7ab01abf784255e4dd09f60c7c77b0 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 23 May 2024 08:14:51 -0400 Subject: [PATCH 073/203] create interfaces --- .../importer/model/workflow/GermplasmWorkflow.java | 6 ++++++ .../model/workflow/SampleSubmissionWorkflow.java | 6 ++++++ .../brapps/importer/model/workflow/Workflow.java | 11 +++++++++++ 3 files changed, 23 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java new file mode 100644 index 000000000..f26342d3b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java @@ -0,0 +1,6 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +@FunctionalInterface +public interface GermplasmWorkflow extends Workflow { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java new file mode 100644 index 000000000..19be9bdcc --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java @@ -0,0 +1,6 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +@FunctionalInterface +public interface SampleSubmissionWorkflow extends Workflow { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java new file mode 100644 index 000000000..8bc19120e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -0,0 +1,11 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +import io.micronaut.core.order.Ordered; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; + +import java.util.Optional; + +@FunctionalInterface +public interface Workflow extends Ordered { + Optional process(ImportServiceContext context); +} From 0459dcd5944330b4ab03219176a545b6af405fc7 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 24 May 2024 12:25:20 -0400 Subject: [PATCH 074/203] create and update trial middleware --- .../experiment/ExperimentUtilities.java | 51 +++++++++++++++++++ .../AppendOverwritePhenotypesWorkflow.java | 13 ++--- .../middleware/commit/BrAPITrialCreation.java | 50 ++++++++++++++++++ .../middleware/commit/BrAPITrialUpdate.java | 51 +++++++++++++++++++ .../middleware/commit/ChangeCommit.java | 25 +++++++++ .../middleware/commit/CreateBrAPITrials.java | 4 -- .../process/brapi/NewPendingBrAPIObjects.java | 29 +---------- .../read/brapi/RequiredBrAPIData.java | 30 +++++------ .../read/brapi/RequiredDatasets.java | 2 + .../read/brapi/RequiredGermplasm.java | 2 + .../read/brapi/RequiredLocations.java | 2 + .../read/brapi/RequiredObservationUnits.java | 2 + .../read/brapi/RequiredStudies.java | 2 + .../middleware/read/brapi/RequiredTrials.java | 2 + .../model/ExpUnitMiddlewareContext.java | 10 ++-- 15 files changed, 214 insertions(+), 61 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCreation.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdate.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ChangeCommit.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/CreateBrAPITrials.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index e321c2f59..120153fc6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -1,27 +1,78 @@ package org.breedinginsight.brapps.importer.services.processors.experiment; +import com.google.gson.Gson; +import com.google.gson.JsonIOException; import com.google.gson.JsonObject; +import com.google.gson.JsonSyntaxException; import io.micronaut.http.HttpStatus; import io.micronaut.http.exceptions.HttpStatusException; +import io.reactivex.functions.Function; import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.checkerframework.checker.units.qual.C; +import javax.inject.Inject; +import javax.inject.Singleton; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.*; import java.util.stream.Collectors; +@Singleton public class ExperimentUtilities { public static final CharSequence COMMA_DELIMITER = ","; public static final String TIMESTAMP_PREFIX = "TS:"; + @Inject + Gson gson; + + public ExperimentUtilities(Gson gson) { + this.gson = gson; + } + + public Optional clone(T obj, Class clazz) { + try { + return Optional.ofNullable(gson.fromJson(gson.toJson(obj), clazz)); + } catch (JsonSyntaxException e) { + return Optional.empty(); + } + } + public List getNewObjects(Map> objectsByName, Class clazz) { + return objectsByName.values().stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW) + .map(PendingImportObject::getBrAPIObject) + .map(b->clone(b, clazz)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + public Map getMutationsByObjectId(Map> objectsByName, Function dbIdFilter, Class clazz) { + return objectsByName.values().stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED) + .map(PendingImportObject::getBrAPIObject) + .map(b->clone(b, clazz)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors + .toMap(brapiObj -> { + try { + return dbIdFilter.apply(brapiObj); + } catch (Exception e) { + throw new RuntimeException(e); + } + }, + brapiObj -> brapiObj)); + } public static List importRowsToExperimentObservations(List importRows) { return importRows.stream() .map(trialImport -> (ExperimentObservation) trialImport) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 96878ba62..5db10d5b7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -18,17 +18,12 @@ public class AppendOverwritePhenotypesWorkflow implements Workflow { ExpUnitMiddleware middleware; - Provider transactionProvider; - Provider validateAllRowsHaveIDsProvider; - Provider getExistingBrAPIDataProvider; @Inject - public AppendOverwritePhenotypesWorkflow(Provider transactionProvider, - Provider validateAllRowsHaveIDsProvider, - Provider getExistingBrAPIDataProvider) { + public AppendOverwritePhenotypesWorkflow(Transaction transaction, + ValidateAllRowsHaveIDs validateAllRowsHaveIDs, + GetExistingBrAPIData getExistingBrAPIData) { - this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(transactionProvider.get(), - validateAllRowsHaveIDsProvider.get(), - getExistingBrAPIDataProvider.get()); + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(transaction, validateAllRowsHaveIDs, getExistingBrAPIData); } @Override public ProcessedData process(ImportContext context) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCreation.java new file mode 100644 index 000000000..f2e906f10 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCreation.java @@ -0,0 +1,50 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Slf4j +@Prototype +public class BrAPITrialCreation extends ExpUnitMiddleware { + + ExperimentUtilities experimentUtilities; + BrAPITrialDAO brapiTrialDAO; + private List newBrAPITrials; + @Inject + public BrAPITrialCreation(ExperimentUtilities experimentUtilities, BrAPITrialDAO brapiTrialDAO) { + this.experimentUtilities = experimentUtilities; + this.brapiTrialDAO = brapiTrialDAO; + } + @Override + public boolean process(ExpUnitMiddlewareContext context) { + newBrAPITrials = experimentUtilities.getNewObjects(context.getPendingData().getTrialByNameNoScope(), BrAPITrial.class); + + try { + List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newBrAPITrials, context.getImportContext().getProgram().getId(), context.getImportContext().getUpload())); + + // Update the context by setting the system-generated dbId for each newly created trial + for (BrAPITrial createdTrial : createdTrials) { + String createdTrialNameNoScope = Utilities.removeProgramKey(createdTrial.getTrialName(), context.getImportContext().getProgram().getKey()); + context.getPendingData().getTrialByNameNoScope().get(createdTrialNameNoScope).getBrAPIObject().setTrialDbId(createdTrial.getTrialDbId()); + } + + } catch (ApiException e) { + throw new RuntimeException(e); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdate.java new file mode 100644 index 000000000..425ca2c08 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdate.java @@ -0,0 +1,51 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Slf4j +@Prototype +public class BrAPITrialUpdate extends ExpUnitMiddleware { + + ExperimentUtilities experimentUtilities; + BrAPITrialDAO brapiTrialDAO; + private Map mutatedTrialsById; + + @Inject + public BrAPITrialUpdate(ExperimentUtilities experimentUtilities, BrAPITrialDAO brapiTrialDAO) { + this.experimentUtilities = experimentUtilities; + this.brapiTrialDAO = brapiTrialDAO; + } + @Override + public boolean process(ExpUnitMiddlewareContext context) { + + mutatedTrialsById = experimentUtilities.getMutationsByObjectId(context.getPendingData().getTrialByNameNoScope(), BrAPITrial::getTrialDbId, BrAPITrial.class); + + mutatedTrialsById.forEach((id, trial) -> { + try { + brapiTrialDAO.updateBrAPITrial(id, trial, context.getImportContext().getProgram().getId()); + } catch (ApiException e) { + log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating dataset observation variables: ", e); + throw new InternalServerException(e.getMessage(), e); + } + }); + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ChangeCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ChangeCommit.java new file mode 100644 index 000000000..514f1d3e1 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ChangeCommit.java @@ -0,0 +1,25 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +import javax.inject.Inject; + +@Slf4j +public class ChangeCommit extends ExpUnitMiddleware { + ExpUnitMiddleware middleware; + @Inject + public ChangeCommit(ExpUnitMiddleware brAPITrialCreation) { + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(brAPITrialCreation); + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + log.debug("starting post of experiment data to BrAPI server"); + + + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/CreateBrAPITrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/CreateBrAPITrials.java deleted file mode 100644 index d6ea6360b..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/CreateBrAPITrials.java +++ /dev/null @@ -1,4 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; - -public class CreateBrAPITrials { -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java index 8f7e6362a..9b45bb386 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java @@ -1,42 +1,17 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi; -import io.micronaut.http.HttpStatus; -import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.brapi.v2.model.pheno.BrAPIObservation; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; -import org.breedinginsight.model.Program; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.jooq.DSLContext; import javax.inject.Inject; -import javax.inject.Provider; -import java.math.BigInteger; -import java.util.*; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.MULTIPLE_EXP_TITLES; @Slf4j public class NewPendingBrAPIObjects extends ExpUnitMiddleware { ExpUnitMiddleware middleware; - Provider pendingObservationProvider; @Inject - public NewPendingBrAPIObjects(Provider pendingObservationProvider) { - this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(pendingObservationProvider.get()); // Construct new pending observation + public NewPendingBrAPIObjects(PendingObservation pendingObservation) { + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(pendingObservation); // Construct new pending observation } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java index 180e6957a..0e5ee2a6e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java @@ -10,28 +10,22 @@ @Slf4j public class RequiredBrAPIData extends ExpUnitMiddleware { ExpUnitMiddleware middleware; - Provider requiredObservationUnitsProvider; - Provider requiredTrialsProvider; - Provider requiredStudiesProvider; - Provider requiredLocationsProvider; - Provider requiredDatasetsProvider; - Provider requiredGermplasmProvider; @Inject - public RequiredBrAPIData(Provider requiredObservationUnitsProvider, - Provider requiredTrialsProvider, - Provider requiredStudiesProvider, - Provider requiredLocationsProvider, - Provider requiredDatasetsProvider, - Provider requiredGermplasmProvider) { + public RequiredBrAPIData(RequiredObservationUnits requiredObservationUnits, + RequiredTrials requiredTrials, + RequiredStudies requiredStudies, + RequiredLocations requiredLocations, + RequiredDatasets requiredDatasets, + RequiredGermplasm requiredGermplasm) { this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( - requiredObservationUnitsProvider.get(), // Fetch the BrAPI units for the required exp unit ids - requiredTrialsProvider.get(), // Fetch the BrAPI trials belonging to the exp units - requiredStudiesProvider.get(), // Fetch the BrAPI studies belonging to the exp units - requiredLocationsProvider.get(), // Fetch the BrAPI locations belonging to the exp units - requiredDatasetsProvider.get(), // Fetch the dataset belonging to the exp units - requiredGermplasmProvider.get()); // Fetch the germplasm belonging to the exp units + requiredObservationUnits, // Fetch the BrAPI units for the required exp unit ids + requiredTrials, // Fetch the BrAPI trials belonging to the exp units + requiredStudies, // Fetch the BrAPI studies belonging to the exp units + requiredLocations, // Fetch the BrAPI locations belonging to the exp units + requiredDatasets, // Fetch the dataset belonging to the exp units + requiredGermplasm); // Fetch the germplasm belonging to the exp units } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java index a2536ecf0..36e8be04e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; +import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; @@ -17,6 +18,7 @@ import java.util.Map; @Slf4j +@Prototype public class RequiredDatasets extends ExpUnitMiddleware { private final DatasetService datasetService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java index 1d40ea781..56393ce8d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; +import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; @@ -20,6 +21,7 @@ import java.util.stream.Collectors; @Slf4j +@Prototype public class RequiredGermplasm extends ExpUnitMiddleware { private final GermplasmService germplasmService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java index fed9bf455..49f3ba54c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; +import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; @@ -21,6 +22,7 @@ import java.util.stream.Collectors; @Slf4j +@Prototype public class RequiredLocations extends ExpUnitMiddleware { LocationService locationService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java index ba19b96f3..e9451e9b0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; +import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservationUnit; @@ -15,6 +16,7 @@ import java.util.stream.Collectors; @Slf4j +@Prototype public class RequiredObservationUnits extends ExpUnitMiddleware { ObservationUnitService observationUnitService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java index 493ae0021..d780d9fc6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; +import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; @@ -21,6 +22,7 @@ import java.util.stream.Collectors; @Slf4j +@Prototype public class RequiredStudies extends ExpUnitMiddleware { StudyService studyService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java index aac07d77c..9deb5f2b0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; +import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; @@ -17,6 +18,7 @@ import java.util.stream.Collectors; @Slf4j +@Prototype public class RequiredTrials extends ExpUnitMiddleware { TrialService trialService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java index 1e81ff833..f50c82ad9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java @@ -1,13 +1,17 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.model; -import lombok.*; +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; - -@Data +@Getter +@Setter @Builder public class ExpUnitMiddlewareContext { + private ImportContext importContext; private ExpUnitContext expUnitContext; private PendingData pendingData; + } From 3b8c4ca45a150533bcd0914a22709ad591fff9f0 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 24 May 2024 12:35:40 -0400 Subject: [PATCH 075/203] update observation middleware --- .../commit/BrAPIObservationUpdate.java | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUpdate.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUpdate.java new file mode 100644 index 000000000..ada8556e5 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUpdate.java @@ -0,0 +1,71 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.Map; +import java.util.Objects; + +@Slf4j +@Prototype +public class BrAPIObservationUpdate extends ExpUnitMiddleware { + + ExperimentUtilities experimentUtilities; + BrAPIObservationDAO brapiObservationDAO; + private Map mutatedObservationByDbId; + + @Inject + public BrAPIObservationUpdate(ExperimentUtilities experimentUtilities, BrAPIObservationDAO brapiObservationDAO) { + this.experimentUtilities = experimentUtilities; + this.brapiObservationDAO = brapiObservationDAO; + } + @Override + public boolean process(ExpUnitMiddlewareContext context) { + + mutatedObservationByDbId = experimentUtilities.getMutationsByObjectId(context.getPendingData().getPendingObservationByHash(), BrAPIObservation::getObservationDbId, BrAPIObservation.class); + + mutatedObservationByDbId.forEach((id, observation) -> { + try { + if (observation == null) { + throw new Exception("Null observation"); + } + BrAPIObservation updatedObs = brapiObservationDAO.updateBrAPIObservation(id, observation, context.getImportContext().getProgram().getId()); + + if (updatedObs == null) { + throw new Exception("Null updated observation"); + } + + if (!Objects.equals(observation.getValue(), updatedObs.getValue()) + || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { + String message; + if(!Objects.equals(observation.getValue(), updatedObs.getValue())) { + message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); + } else { + message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); + } + throw new Exception(message); + } + } catch (ApiException e) { + log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating observation: ", e); + throw new InternalServerException(e.getMessage(), e); + } + }); + + return processNext(context); + } +} From e237fe9462e417b63b2b7368c5130e7f1a5324c3 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 24 May 2024 13:24:58 -0400 Subject: [PATCH 076/203] create dataset middleware --- .../commit/BrAPIDatasetCreation.java | 59 +++++++++++++++++++ .../middleware/commit/BrAPITrialCreation.java | 2 +- 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCreation.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCreation.java new file mode 100644 index 000000000..92d8bf226 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCreation.java @@ -0,0 +1,59 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.request.BrAPIListNewRequest; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +@Slf4j +@Prototype +public class BrAPIDatasetCreation extends ExpUnitMiddleware { + + ExperimentUtilities experimentUtilities; + BrAPIListDAO brapiListDAO; + private List newDatasetRequests; + @Inject + public BrAPIDatasetCreation(ExperimentUtilities experimentUtilities, BrAPIListDAO brapiListDAO) { + this.experimentUtilities = experimentUtilities; + this.brapiListDAO = brapiListDAO; + } + @Override + public boolean process(ExpUnitMiddlewareContext context) { + newDatasetRequests = experimentUtilities.getNewObjects(context.getPendingData().getObsVarDatasetByName(), BrAPIListDetails.class).stream().map(details -> { + BrAPIListNewRequest request = new BrAPIListNewRequest(); + request.setListName(details.getListName()); + request.setListType(details.getListType()); + request.setExternalReferences(details.getExternalReferences()); + request.setAdditionalInfo(details.getAdditionalInfo()); + request.data(details.getData()); + return request; + }).collect(Collectors.toList()); + + List createdDatasets = null; + try { + createdDatasets = new ArrayList<>(brapiListDAO.createBrAPILists(newDatasetRequests, context.getImportContext().getProgram().getId(), context.getImportContext().getUpload())); + + // Update the context cache by setting the system-generated dbId for each newly created dataset + createdDatasets.forEach(summary -> context.getPendingData().getObsVarDatasetByName().get(summary.getListName()).getBrAPIObject().setListDbId(summary.getListDbId())); + } catch (ApiException e) { + throw new RuntimeException(e); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCreation.java index f2e906f10..dbcd6981b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCreation.java @@ -35,7 +35,7 @@ public boolean process(ExpUnitMiddlewareContext context) { try { List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newBrAPITrials, context.getImportContext().getProgram().getId(), context.getImportContext().getUpload())); - // Update the context by setting the system-generated dbId for each newly created trial + // Update the context cache by setting the system-generated dbId for each newly created trial for (BrAPITrial createdTrial : createdTrials) { String createdTrialNameNoScope = Utilities.removeProgramKey(createdTrial.getTrialName(), context.getImportContext().getProgram().getKey()); context.getPendingData().getTrialByNameNoScope().get(createdTrialNameNoScope).getBrAPIObject().setTrialDbId(createdTrial.getTrialDbId()); From 8f483ff2e2344233a663050f948c121b6c936951 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 24 May 2024 16:33:58 -0400 Subject: [PATCH 077/203] create location creation middleware --- .../{ChangeCommit.java => BrAPICommit.java} | 4 +- .../commit/BrAPILocationCreation.java | 81 +++++++++++++++++++ 2 files changed, 83 insertions(+), 2 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/{ChangeCommit.java => BrAPICommit.java} (86%) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreation.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ChangeCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java similarity index 86% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ChangeCommit.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java index 514f1d3e1..3efda4cce 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ChangeCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java @@ -7,10 +7,10 @@ import javax.inject.Inject; @Slf4j -public class ChangeCommit extends ExpUnitMiddleware { +public class BrAPICommit extends ExpUnitMiddleware { ExpUnitMiddleware middleware; @Inject - public ChangeCommit(ExpUnitMiddleware brAPITrialCreation) { + public BrAPICommit(ExpUnitMiddleware brAPITrialCreation) { this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(brAPITrialCreation); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreation.java new file mode 100644 index 000000000..1b23b9883 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreation.java @@ -0,0 +1,81 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.request.BrAPIListNewRequest; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; +import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.ProgramLocationService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +@Slf4j +@Prototype +public class BrAPILocationCreation extends ExpUnitMiddleware { + + ExperimentUtilities experimentUtilities; + ProgramLocationService locationService; + private List newLocations; + @Inject + public BrAPILocationCreation(ExperimentUtilities experimentUtilities, ProgramLocationService locationService) { + this.experimentUtilities = experimentUtilities; + this.locationService = locationService; + } + @Override + public boolean process(ExpUnitMiddlewareContext context) { + newLocations = experimentUtilities.getNewObjects(context.getPendingData().getLocationByName(), ProgramLocation.class) + .stream() + .map(location -> ProgramLocationRequest.builder() + .name(location.getName()) + .build()) + .collect(Collectors.toList()); + + // Create acting user + AuthenticatedUser actingUser = new AuthenticatedUser(context.getImportContext().getUpload().getUpdatedByUser().getName(), new ArrayList<>(), context.getImportContext().getUpload().getUpdatedByUser().getId(), new ArrayList<>()); + + // Create new locations in brapi service + List createdLocations = null; + try { + createdLocations = new ArrayList<>(locationService.create(actingUser, context.getImportContext().getProgram().getId(), newLocations)); + + // Update the context cache + for (ProgramLocation createdLocation : createdLocations) { + + // Set the system-generated dbId for each newly created location + context.getPendingData().getLocationByName().get(createdLocation.getName()).getBrAPIObject().setLocationDbId(createdLocation.getLocationDbId()); + + // Set the location dbid for cached studies + context.getPendingData().getStudyByNameNoScope().values().stream() + .filter(study -> createdLocation.getId().toString().equals(study.getBrAPIObject().getLocationDbId())) + .forEach(study -> study.getBrAPIObject().setLocationDbId(createdLocation.getLocationDbId())); + } + + } catch (MissingRequiredInfoException e) { + throw new RuntimeException(e); + } catch (UnprocessableEntityException e) { + throw new RuntimeException(e); + } catch (DoesNotExistException e) { + throw new RuntimeException(e); + } + + return processNext(context); + } +} From 322a69d6578f1b922c45032ca8c380e74cfca011 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 27 May 2024 21:46:27 -0400 Subject: [PATCH 078/203] create brapi command interface --- .../middleware/commit/BrAPIAction.java | 11 ++++ .../middleware/commit/BrAPICreation.java | 51 ++++++++++++++++ .../commit/BrAPIDatasetCreation.java | 2 + .../commit/BrAPILocationCreation.java | 1 + .../middleware/commit/BrAPIState.java | 4 ++ .../middleware/commit/BrAPITrialUpdate.java | 5 +- .../middleware/commit/BrAPIUpdate.java | 60 +++++++++++++++++++ .../commit/ExperimentImportEntity.java | 22 +++++++ .../middleware/read/brapi/BrAPIRead.java | 39 ++++++++++++ 9 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIAction.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICreation.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIState.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIUpdate.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ExperimentImportEntity.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/BrAPIRead.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIAction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIAction.java new file mode 100644 index 000000000..418591aca --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIAction.java @@ -0,0 +1,11 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +import java.util.List; +import java.util.Optional; + +public interface BrAPIAction { + Optional execute(); + ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICreation.java new file mode 100644 index 000000000..49fa7284b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICreation.java @@ -0,0 +1,51 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +import java.util.List; +import java.util.Optional; +@Slf4j +public abstract class BrAPICreation implements BrAPIAction { + ExperimentImportEntity entity; + + BrAPICreation(ExpUnitMiddlewareContext context) { + this.entity = getEntity(context); + } + + public Optional execute() { + List newMembers = entity.getNewBrAPIMembers(); + try { + List createdMembers = entity.brapiPost(newMembers); + return Optional.of(new BrAPICreationState(createdMembers)); + } catch (ApiException e) { + // TODO: add specific error messages to entity service + log.error("Error creating..."); + throw new InternalServerException("Error creating...", e); + } + } + + @Getter + public class BrAPICreationState implements BrAPIState { + + private final List members; + + public BrAPICreationState(List createdMembers) { + this.members = createdMembers; + } + + public boolean undo() { + List createdMembers = this.getMembers(); + try { + return entity.brapiDelete(createdMembers); + } catch (ApiException e) { + // TODO: add specific error messages to entity service + log.error("Error deleting..."); + throw new InternalServerException("Error deleting...", e); + } + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCreation.java index 92d8bf226..eafca0b4b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCreation.java @@ -34,6 +34,7 @@ public BrAPIDatasetCreation(ExperimentUtilities experimentUtilities, BrAPIListDA } @Override public boolean process(ExpUnitMiddlewareContext context) { + // Construct request newDatasetRequests = experimentUtilities.getNewObjects(context.getPendingData().getObsVarDatasetByName(), BrAPIListDetails.class).stream().map(details -> { BrAPIListNewRequest request = new BrAPIListNewRequest(); request.setListName(details.getListName()); @@ -46,6 +47,7 @@ public boolean process(ExpUnitMiddlewareContext context) { List createdDatasets = null; try { + // Create entities in brapi service createdDatasets = new ArrayList<>(brapiListDAO.createBrAPILists(newDatasetRequests, context.getImportContext().getProgram().getId(), context.getImportContext().getUpload())); // Update the context cache by setting the system-generated dbId for each newly created dataset diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreation.java index 1b23b9883..5b266be09 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreation.java @@ -41,6 +41,7 @@ public BrAPILocationCreation(ExperimentUtilities experimentUtilities, ProgramLoc } @Override public boolean process(ExpUnitMiddlewareContext context) { + // Construct request newLocations = experimentUtilities.getNewObjects(context.getPendingData().getLocationByName(), ProgramLocation.class) .stream() .map(location -> ProgramLocationRequest.builder() diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIState.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIState.java new file mode 100644 index 000000000..3a85985bd --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIState.java @@ -0,0 +1,4 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +public interface BrAPIState { +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdate.java index 425ca2c08..6688065bf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdate.java @@ -31,11 +31,12 @@ public BrAPITrialUpdate(ExperimentUtilities experimentUtilities, BrAPITrialDAO b } @Override public boolean process(ExpUnitMiddlewareContext context) { - - mutatedTrialsById = experimentUtilities.getMutationsByObjectId(context.getPendingData().getTrialByNameNoScope(), BrAPITrial::getTrialDbId, BrAPITrial.class); + // Construct request + mutatedTrialsById = experimentUtilities.getMutationsByObjectId(context.getPendingData().getTrialByNameNoScope(), BrAPITrial::getTrialDbId, BrAPITrial.class); mutatedTrialsById.forEach((id, trial) -> { try { + // Update entities in the brapi service brapiTrialDAO.updateBrAPITrial(id, trial, context.getImportContext().getProgram().getId()); } catch (ApiException e) { log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIUpdate.java new file mode 100644 index 000000000..b3a39741a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIUpdate.java @@ -0,0 +1,60 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +import java.util.List; +import java.util.Optional; + +@Slf4j +public abstract class BrAPIUpdate implements BrAPIAction { + ExperimentImportEntity entity; + + BrAPIUpdate(ExpUnitMiddlewareContext context) { + this.entity = getEntity(context); + } + + public Optional execute() { + return saveAndUpdateCache(entity.getMutatedBrAPIMembers()); + } + + public Optional> getBrAPIState() { + try { + return Optional.of(new BrAPIUpdateState(entity.getBrAPIStateMutatedMembers())); + } catch (ApiException e) { + // TODO: add specific error messages to entity service + log.error("Error getting..."); + throw new InternalServerException("Error getting...", e); + } + + } + + protected Optional saveAndUpdateCache(List members) { + return Optional.ofNullable(members).map(changes -> { + try { + List savedMembers = entity.brapiPut(changes); + entity.updateCache(savedMembers); + return new BrAPIUpdateState(savedMembers); + } catch (ApiException e) { + // TODO: add specific error messages to entity service + log.error("Error updating..."); + throw new InternalServerException("Error updating...", e); + } + }); + } + + @Getter + public class BrAPIUpdateState implements BrAPIState { + private final List members; + + public BrAPIUpdateState(List existingMembers) { this.members = existingMembers; } + + public boolean restore() { + return saveAndUpdateCache(this.getMembers()).isPresent(); + } + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ExperimentImportEntity.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ExperimentImportEntity.java new file mode 100644 index 000000000..77900af17 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ExperimentImportEntity.java @@ -0,0 +1,22 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; + +import java.util.List; +import java.util.Map; + +public interface ExperimentImportEntity { + public List constructUpdate(PendingData cache); + public List constructNew(PendingData cache); + public Map> getBrapiState(); + public List brapiPost(List members) throws ApiException; + public List brapiRead() throws ApiException; + public List brapiPut(List members) throws ApiException; + public boolean brapiDelete(List members) throws ApiException; + public List getBrAPIStateMutatedMembers() throws ApiException; + public List getMutatedBrAPIMembers(); + + public List getNewBrAPIMembers(); + public void updateCache(List members); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/BrAPIRead.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/BrAPIRead.java new file mode 100644 index 000000000..141e48850 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/BrAPIRead.java @@ -0,0 +1,39 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.BrAPIAction; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +import java.util.List; +import java.util.Optional; + +@Slf4j +public abstract class BrAPIRead implements BrAPIAction { + ExperimentImportEntity entity; + + BrAPIRead(ExpUnitMiddlewareContext context) { + this.entity = getEntity(context); + } + public Optional execute() { + try { + List fetchedMembers = entity.brapiRead(); + entity.updateCache(fetchedMembers); + return Optional.of(new BrAPIReadState(fetchedMembers)); + } catch(ApiException e) { + // TODO: add specific error messages to entity service + log.error("Error reading..."); + throw new InternalServerException("Error reading...", e); + } + } + @Getter + public static class BrAPIReadState implements BrAPIState { + + private final List members; + public BrAPIReadState(List fetchedMembers) { this.members = fetchedMembers; } + } +} From a4fdc48a2b029aeaae9078d6dcf4a3d9d776efac Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 28 May 2024 14:44:03 -0400 Subject: [PATCH 079/203] create PendingTrial class --- .../experiment/ExperimentUtilities.java | 32 ++++++- .../appendoverwrite/entity/PendingTrial.java | 94 +++++++++++++++++++ .../middleware/commit/BrAPICreation.java | 5 +- .../middleware/commit/BrAPIUpdate.java | 8 +- .../commit/ExperimentImportEntity.java | 18 ++-- .../middleware/read/brapi/BrAPIRead.java | 3 +- .../appendoverwrite/model/ExpUnitContext.java | 12 +++ 7 files changed, 151 insertions(+), 21 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 120153fc6..17a366872 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -1,14 +1,12 @@ package org.breedinginsight.brapps.importer.services.processors.experiment; import com.google.gson.Gson; -import com.google.gson.JsonIOException; import com.google.gson.JsonObject; import com.google.gson.JsonSyntaxException; import io.micronaut.http.HttpStatus; import io.micronaut.http.exceptions.HttpStatusException; import io.reactivex.functions.Function; import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; @@ -18,12 +16,9 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.checkerframework.checker.units.qual.C; import javax.inject.Inject; import javax.inject.Singleton; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; import java.util.*; import java.util.stream.Collectors; @@ -39,6 +34,13 @@ public class ExperimentUtilities { public ExperimentUtilities(Gson gson) { this.gson = gson; } + public boolean isPopulated(List list, Class clazz) { + // Check if the input list is of type + if (list == null || list.isEmpty() || !clazz.isInstance(list.get(0))) { + return false; + } + return true; + } public Optional clone(T obj, Class clazz) { try { @@ -56,6 +58,26 @@ public List getNewObjects(Map> objectsByName .map(Optional::get) .collect(Collectors.toList()); } + public List copyMutationsFromCache(Map> pendingCacheMap, Class clazz) { + return pendingCacheMap.values().stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED) + .map(PendingImportObject::getBrAPIObject) + .map(b->clone(b, clazz)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + public List copyWorkflowCachePendingBrAPIObjects(Map> pendingCacheMap, + Class clazz, + ImportObjectState status) { + return pendingCacheMap.values().stream() + .filter(preview -> preview != null && preview.getState() == status) + .map(PendingImportObject::getBrAPIObject) + .map(b->clone(b, clazz)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } public Map getMutationsByObjectId(Map> objectsByName, Function dbIdFilter, Class clazz) { return objectsByName.values().stream() .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java new file mode 100644 index 000000000..9f155a93b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java @@ -0,0 +1,94 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; + +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +public class PendingTrial implements ExperimentImportEntity { + ExpUnitContext cache; + ImportContext importContext; + @Inject + TrialService trialService; + BrAPITrialDAO brapiTrialDAO; + @Inject + ExperimentUtilities experimentUtilities; + + public PendingTrial(ExpUnitMiddlewareContext context, + TrialService trialService, + BrAPITrialDAO brapiTrialDAO, + ExperimentUtilities experimentUtilities) { + this.cache = context.getExpUnitContext(); + this.importContext = context.getImportContext(); + this.trialService = trialService; + this.brapiTrialDAO = brapiTrialDAO; + this.experimentUtilities = experimentUtilities; + + } + @Override + public List brapiPost(List members) throws ApiException { + return brapiTrialDAO.createBrAPITrials(members, importContext.getProgram().getId(), importContext.getUpload()); + } + @Override + public List brapiRead() throws ApiException { + return null; + } + @Override + public List brapiPut(List members) throws ApiException, IllegalArgumentException { + // Check if the input list is of type List + if (!experimentUtilities.isPopulated(members, BrAPITrial.class)) { + return new ArrayList(); + } + + List updatedTrials = new ArrayList<>(); + for (U member : members) { + BrAPITrial trial = (BrAPITrial) member; + Optional.ofNullable(brapiTrialDAO.updateBrAPITrial(trial.getTrialDbId(), trial, importContext.getProgram().getId())).ifPresent(updatedTrials::add); + } + + return (List) updatedTrials; + } + @Override + public boolean brapiDelete(List members) throws ApiException { + // TODO: implement delete for trials on BrapiJavaTestServer + return false; + } + @Override + public List getBrAPIState(ImportObjectState status) throws ApiException { + List ids = copyWorkflowMembers(status).stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList()); + return brapiTrialDAO.getTrialsByDbIds(ids, importContext.getProgram()); + } + @Override + public List copyWorkflowMembers(ImportObjectState status) { + return experimentUtilities.copyWorkflowCachePendingBrAPIObjects(cache.getTrialByNameNoScope(), BrAPITrial.class, status); + } + + @Override + public void updateWorkflowWithDbId(List members) { + // Check if the input list is of type List + if (!experimentUtilities.isPopulated(members, BrAPITrial.class)) { + return; + } + + // Update the workflow ref by setting the system-generated dbId for each newly created trial + for (U member : members) { + BrAPITrial trial = (BrAPITrial) member; + String createdTrialNameNoScope = Utilities.removeProgramKey(trial.getTrialName(), importContext.getProgram().getKey()); + cache.getTrialByNameNoScope().get(createdTrialNameNoScope).getBrAPIObject().setTrialDbId(trial.getTrialDbId()); + } + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICreation.java index 49fa7284b..2702e0b4a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICreation.java @@ -4,6 +4,8 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import java.util.List; @@ -17,9 +19,10 @@ public abstract class BrAPICreation implements BrAPIAction { } public Optional execute() { - List newMembers = entity.getNewBrAPIMembers(); + List newMembers = entity.copyWorkflowMembers(ImportObjectState.NEW); try { List createdMembers = entity.brapiPost(newMembers); + entity.updateWorkflowWithDbId(createdMembers); return Optional.of(new BrAPICreationState(createdMembers)); } catch (ApiException e) { // TODO: add specific error messages to entity service diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIUpdate.java index b3a39741a..b4cfd2e33 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIUpdate.java @@ -4,6 +4,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import java.util.List; @@ -18,12 +19,12 @@ public abstract class BrAPIUpdate implements BrAPIAction { } public Optional execute() { - return saveAndUpdateCache(entity.getMutatedBrAPIMembers()); + return saveAndUpdateCache(entity.copyWorkflowMembers(ImportObjectState.MUTATED)); } public Optional> getBrAPIState() { try { - return Optional.of(new BrAPIUpdateState(entity.getBrAPIStateMutatedMembers())); + return Optional.of(new BrAPIUpdateState(entity.getBrAPIState(ImportObjectState.MUTATED))); } catch (ApiException e) { // TODO: add specific error messages to entity service log.error("Error getting..."); @@ -36,7 +37,8 @@ protected Optional saveAndUpdateCache(List members) { return Optional.ofNullable(members).map(changes -> { try { List savedMembers = entity.brapiPut(changes); - entity.updateCache(savedMembers); + // TODO: set updated fields of workflow brapi object + // entity.updateWorkflow(savedMembers); return new BrAPIUpdateState(savedMembers); } catch (ApiException e) { // TODO: add specific error messages to entity service diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ExperimentImportEntity.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ExperimentImportEntity.java index 77900af17..4da28d4df 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ExperimentImportEntity.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ExperimentImportEntity.java @@ -1,22 +1,18 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import java.util.List; -import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Function; public interface ExperimentImportEntity { - public List constructUpdate(PendingData cache); - public List constructNew(PendingData cache); - public Map> getBrapiState(); public List brapiPost(List members) throws ApiException; public List brapiRead() throws ApiException; - public List brapiPut(List members) throws ApiException; + public List brapiPut(List members) throws ApiException, IllegalArgumentException; public boolean brapiDelete(List members) throws ApiException; - public List getBrAPIStateMutatedMembers() throws ApiException; - public List getMutatedBrAPIMembers(); - - public List getNewBrAPIMembers(); - public void updateCache(List members); + public List getBrAPIState(ImportObjectState status) throws ApiException; + public List copyWorkflowMembers(ImportObjectState status); + public void updateWorkflowWithDbId(List members); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/BrAPIRead.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/BrAPIRead.java index 141e48850..730bb94da 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/BrAPIRead.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/BrAPIRead.java @@ -22,7 +22,8 @@ public abstract class BrAPIRead implements BrAPIAction { public Optional execute() { try { List fetchedMembers = entity.brapiRead(); - entity.updateCache(fetchedMembers); + // TODO: update workflow ref by creating hashmaps + //entity.updateWorkflow(fetchedMembers); return Optional.of(new BrAPIReadState(fetchedMembers)); } catch(ApiException e) { // TODO: add specific error messages to entity service diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java index 6988cbf89..29aef8a6a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java @@ -6,6 +6,7 @@ import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -28,4 +29,15 @@ public class ExpUnitContext { private Map> pendingLocationByOUId = new HashMap<>(); private Map> pendingGermplasmByOUId = new HashMap<>(); + // Carry over from PendingData + private Map> observationUnitByNameNoScope; + private Map> trialByNameNoScope; + private Map> studyByNameNoScope; + private Map> locationByName; + private Map> obsVarDatasetByName; + private Map> existingGermplasmByGID; + private Map> pendingObservationByHash; + private Map> timeStampColByPheno; + private Map existingObsByObsHash; + private ValidationErrors validationErrors; } From 2345cffe35fd3271f60e169b02f48f7c09b4e91c Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 28 May 2024 18:17:00 -0400 Subject: [PATCH 080/203] add compensate method to brapi trial middleware --- .../commit => action}/BrAPIAction.java | 7 ++- .../commit => action}/BrAPICreation.java | 10 ++-- .../read/brapi => action}/BrAPIRead.java | 10 ++-- .../commit => action}/BrAPIState.java | 2 +- .../action/BrAPITrialCreation.java | 20 +++++++ .../action/BrAPITrialUpdate.java | 20 +++++++ .../commit => action}/BrAPIUpdate.java | 7 ++- .../ExperimentImportEntity.java | 2 +- .../appendoverwrite/entity/PendingTrial.java | 10 +--- .../middleware/commit/BrAPITrial.java | 57 +++++++++++++++++++ .../middleware/commit/BrAPITrialCreation.java | 50 ---------------- ...e.java => BrAPITrialUpdateMiddleware.java} | 6 +- 12 files changed, 120 insertions(+), 81 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{middleware/commit => action}/BrAPIAction.java (53%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{middleware/commit => action}/BrAPICreation.java (82%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{middleware/read/brapi => action}/BrAPIRead.java (74%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{middleware/commit => action}/BrAPIState.java (67%) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPITrialCreation.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPITrialUpdate.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{middleware/commit => action}/BrAPIUpdate.java (88%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{middleware/commit => entity}/ExperimentImportEntity.java (94%) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrial.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCreation.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/{BrAPITrialUpdate.java => BrAPITrialUpdateMiddleware.java} (91%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIAction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIAction.java similarity index 53% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIAction.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIAction.java index 418591aca..201139a5f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIAction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIAction.java @@ -1,11 +1,12 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import java.util.List; import java.util.Optional; public interface BrAPIAction { - Optional execute(); + Optional execute() throws ApiException; ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPICreation.java similarity index 82% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICreation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPICreation.java index 2702e0b4a..9ae86c1a8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPICreation.java @@ -1,11 +1,11 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPITrial; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import java.util.List; @@ -14,11 +14,13 @@ public abstract class BrAPICreation implements BrAPIAction { ExperimentImportEntity entity; - BrAPICreation(ExpUnitMiddlewareContext context) { + protected BrAPICreation(ExpUnitMiddlewareContext context) { this.entity = getEntity(context); } - public Optional execute() { + public abstract ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context); + + public Optional execute() throws ApiException { List newMembers = entity.copyWorkflowMembers(ImportObjectState.NEW); try { List createdMembers = entity.brapiPost(newMembers); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/BrAPIRead.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIRead.java similarity index 74% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/BrAPIRead.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIRead.java index 730bb94da..199288c32 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/BrAPIRead.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIRead.java @@ -1,12 +1,10 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.BrAPIAction; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.BrAPIState; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import java.util.List; @@ -16,10 +14,10 @@ public abstract class BrAPIRead implements BrAPIAction { ExperimentImportEntity entity; - BrAPIRead(ExpUnitMiddlewareContext context) { + protected BrAPIRead(ExpUnitMiddlewareContext context) { this.entity = getEntity(context); } - public Optional execute() { + public Optional execute() throws ApiException { try { List fetchedMembers = entity.brapiRead(); // TODO: update workflow ref by creating hashmaps diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIState.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIState.java similarity index 67% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIState.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIState.java index 3a85985bd..fa9355e6d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIState.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIState.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action; public interface BrAPIState { } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPITrialCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPITrialCreation.java new file mode 100644 index 000000000..0a644b1e3 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPITrialCreation.java @@ -0,0 +1,20 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingTrial; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +@Slf4j +public class BrAPITrialCreation extends BrAPICreation { + + public BrAPITrialCreation(ExpUnitMiddlewareContext context) { + super(context); + } + + @Override + public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + return new PendingTrial(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPITrialUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPITrialUpdate.java new file mode 100644 index 000000000..8c8b136cb --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPITrialUpdate.java @@ -0,0 +1,20 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingTrial; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +@Slf4j +public class BrAPITrialUpdate extends BrAPIUpdate { + + public BrAPITrialUpdate(ExpUnitMiddlewareContext context) { + super(context); + } + + @Override + public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + return new PendingTrial(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIUpdate.java similarity index 88% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIUpdate.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIUpdate.java index b4cfd2e33..ad05553ed 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIUpdate.java @@ -1,10 +1,11 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import java.util.List; @@ -14,11 +15,11 @@ public abstract class BrAPIUpdate implements BrAPIAction { ExperimentImportEntity entity; - BrAPIUpdate(ExpUnitMiddlewareContext context) { + protected BrAPIUpdate(ExpUnitMiddlewareContext context) { this.entity = getEntity(context); } - public Optional execute() { + public Optional execute() throws ApiException { return saveAndUpdateCache(entity.copyWorkflowMembers(ImportObjectState.MUTATED)); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ExperimentImportEntity.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java similarity index 94% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ExperimentImportEntity.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java index 4da28d4df..3a23cb48a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/ExperimentImportEntity.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java index 9f155a93b..58869be54 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java @@ -5,7 +5,6 @@ import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; @@ -27,16 +26,9 @@ public class PendingTrial implements ExperimentImportEntity { @Inject ExperimentUtilities experimentUtilities; - public PendingTrial(ExpUnitMiddlewareContext context, - TrialService trialService, - BrAPITrialDAO brapiTrialDAO, - ExperimentUtilities experimentUtilities) { + public PendingTrial(ExpUnitMiddlewareContext context) { this.cache = context.getExpUnitContext(); this.importContext = context.getImportContext(); - this.trialService = trialService; - this.brapiTrialDAO = brapiTrialDAO; - this.experimentUtilities = experimentUtilities; - } @Override public List brapiPost(List members) throws ApiException { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrial.java new file mode 100644 index 000000000..31629f7af --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrial.java @@ -0,0 +1,57 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import io.micronaut.context.annotation.Prototype; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPICreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPITrialCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPITrialUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; + +import java.util.Optional; + +@Slf4j +@Prototype +@NoArgsConstructor +public class BrAPITrial extends ExpUnitMiddleware { + private BrAPITrialCreation brAPITrialCreation; + private BrAPITrialUpdate brAPITrialUpdate; + private Optional createdBrAPITrials; + private Optional priorBrAPITrials; + private Optional updatedTrials; + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + try { + brAPITrialCreation = new BrAPITrialCreation(context); + createdBrAPITrials = brAPITrialCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); + brAPITrialUpdate = new BrAPITrialUpdate(context); + priorBrAPITrials = brAPITrialUpdate.getBrAPIState().map(s -> (BrAPIUpdate.BrAPIUpdateState) s); + updatedTrials = brAPITrialUpdate.execute().map(s -> (BrAPIUpdate.BrAPIUpdateState) s); + + } catch (ApiException e) { + throw new RuntimeException(e); + } + + return processNext(context); + } + + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // Tag an error if it occurred in this local transaction + error.tag(this.getClass().getName()); + + // Delete any created trials from the BrAPI service + createdBrAPITrials.ifPresent(BrAPICreation.BrAPICreationState::undo); + + // Revert any changes made to trials in the BrAPI service + priorBrAPITrials.ifPresent(BrAPIUpdate.BrAPIUpdateState::restore); + + // Undo the prior local transaction + return compensatePrior(context, error); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCreation.java deleted file mode 100644 index dbcd6981b..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCreation.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; - -import io.micronaut.context.annotation.Prototype; -import io.micronaut.http.server.exceptions.InternalServerException; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.utilities.Utilities; - -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -@Slf4j -@Prototype -public class BrAPITrialCreation extends ExpUnitMiddleware { - - ExperimentUtilities experimentUtilities; - BrAPITrialDAO brapiTrialDAO; - private List newBrAPITrials; - @Inject - public BrAPITrialCreation(ExperimentUtilities experimentUtilities, BrAPITrialDAO brapiTrialDAO) { - this.experimentUtilities = experimentUtilities; - this.brapiTrialDAO = brapiTrialDAO; - } - @Override - public boolean process(ExpUnitMiddlewareContext context) { - newBrAPITrials = experimentUtilities.getNewObjects(context.getPendingData().getTrialByNameNoScope(), BrAPITrial.class); - - try { - List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newBrAPITrials, context.getImportContext().getProgram().getId(), context.getImportContext().getUpload())); - - // Update the context cache by setting the system-generated dbId for each newly created trial - for (BrAPITrial createdTrial : createdTrials) { - String createdTrialNameNoScope = Utilities.removeProgramKey(createdTrial.getTrialName(), context.getImportContext().getProgram().getKey()); - context.getPendingData().getTrialByNameNoScope().get(createdTrialNameNoScope).getBrAPIObject().setTrialDbId(createdTrial.getTrialDbId()); - } - - } catch (ApiException e) { - throw new RuntimeException(e); - } - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdateMiddleware.java similarity index 91% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdate.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdateMiddleware.java index 6688065bf..cc5152948 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdateMiddleware.java @@ -12,20 +12,18 @@ import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; -import java.util.ArrayList; -import java.util.List; import java.util.Map; @Slf4j @Prototype -public class BrAPITrialUpdate extends ExpUnitMiddleware { +public class BrAPITrialUpdateMiddleware extends ExpUnitMiddleware { ExperimentUtilities experimentUtilities; BrAPITrialDAO brapiTrialDAO; private Map mutatedTrialsById; @Inject - public BrAPITrialUpdate(ExperimentUtilities experimentUtilities, BrAPITrialDAO brapiTrialDAO) { + public BrAPITrialUpdateMiddleware(ExperimentUtilities experimentUtilities, BrAPITrialDAO brapiTrialDAO) { this.experimentUtilities = experimentUtilities; this.brapiTrialDAO = brapiTrialDAO; } From ef417bb7489904dfefb5f80274a30dd1e56e2093 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 29 May 2024 11:21:21 -0400 Subject: [PATCH 081/203] create BrAPIReadWorkflowInitialization class --- .../action/{ => create}/BrAPICreation.java | 4 +- .../{ => create}/BrAPITrialCreation.java | 2 +- .../action/{ => read}/BrAPIRead.java | 6 +-- .../read/BrAPIReadWorkflowInitialization.java | 40 +++++++++++++++++++ .../action/read/BrAPITrialRead.java | 20 ++++++++++ .../BrAPITrialReadWorkflowInitialization.java | 21 ++++++++++ .../action/{ => update}/BrAPITrialUpdate.java | 2 +- .../action/{ => update}/BrAPIUpdate.java | 4 +- .../entity/ExperimentImportEntity.java | 1 + .../appendoverwrite/entity/PendingTrial.java | 31 ++++++++++++-- ...{BrAPITrial.java => BrAPITrialCommit.java} | 14 ++++--- .../middleware/read/brapi/RequiredTrials.java | 33 +++------------ 12 files changed, 134 insertions(+), 44 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/{ => create}/BrAPICreation.java (89%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/{ => create}/BrAPITrialCreation.java (95%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/{ => read}/BrAPIRead.java (84%) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialRead.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialReadWorkflowInitialization.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/{ => update}/BrAPITrialUpdate.java (95%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/{ => update}/BrAPIUpdate.java (90%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/{BrAPITrial.java => BrAPITrialCommit.java} (85%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPICreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java similarity index 89% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPICreation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java index 9ae86c1a8..6135e188a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPICreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java @@ -1,10 +1,12 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPITrialCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPITrialCreation.java similarity index 95% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPITrialCreation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPITrialCreation.java index 0a644b1e3..4d0f7aa2c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPITrialCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPITrialCreation.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.core.BrAPITrial; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIRead.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIRead.java similarity index 84% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIRead.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIRead.java index 199288c32..3515e9dec 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIRead.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIRead.java @@ -1,9 +1,11 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; @@ -20,8 +22,6 @@ protected BrAPIRead(ExpUnitMiddlewareContext context) { public Optional execute() throws ApiException { try { List fetchedMembers = entity.brapiRead(); - // TODO: update workflow ref by creating hashmaps - //entity.updateWorkflow(fetchedMembers); return Optional.of(new BrAPIReadState(fetchedMembers)); } catch(ApiException e) { // TODO: add specific error messages to entity service diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java new file mode 100644 index 000000000..364049996 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java @@ -0,0 +1,40 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; + +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +import java.util.List; +import java.util.Optional; + +@Slf4j +public abstract class BrAPIReadWorkflowInitialization implements BrAPIAction { + ExperimentImportEntity entity; + + protected BrAPIReadWorkflowInitialization(ExpUnitMiddlewareContext context) { + this.entity = getEntity(context); + } + + public Optional execute() throws ApiException { + try { + List fetchedMembers = entity.brapiRead(); + entity.initializeWorkflow(fetchedMembers); + return Optional.of(new BrAPIReadState(fetchedMembers)); + } catch(ApiException e) { + // TODO: add specific error messages to entity service + log.error("Error reading..."); + throw new InternalServerException("Error reading...", e); + } + } + @Getter + public static class BrAPIReadState implements BrAPIState { + + private final List members; + public BrAPIReadState(List fetchedMembers) { this.members = fetchedMembers; } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialRead.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialRead.java new file mode 100644 index 000000000..525f4ec9c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialRead.java @@ -0,0 +1,20 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingTrial; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +@Slf4j +public class BrAPITrialRead extends BrAPIRead { + + public BrAPITrialRead(ExpUnitMiddlewareContext context) { + super(context); + } + + @Override + public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + return new PendingTrial(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialReadWorkflowInitialization.java new file mode 100644 index 000000000..e3aeb2da4 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialReadWorkflowInitialization.java @@ -0,0 +1,21 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.BrAPIReadWorkflowInitialization; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingTrial; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +@Slf4j +public class BrAPITrialReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { + + public BrAPITrialReadWorkflowInitialization(ExpUnitMiddlewareContext context) { + super(context); + } + + @Override + public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + return new PendingTrial(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPITrialUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPITrialUpdate.java similarity index 95% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPITrialUpdate.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPITrialUpdate.java index 8c8b136cb..7303eef32 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPITrialUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPITrialUpdate.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.core.BrAPITrial; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdate.java similarity index 90% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIUpdate.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdate.java index ad05553ed..a4928453f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdate.java @@ -1,10 +1,12 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java index 3a23cb48a..daa0e1a70 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java @@ -15,4 +15,5 @@ public interface ExperimentImportEntity { public List getBrAPIState(ImportObjectState status) throws ApiException; public List copyWorkflowMembers(ImportObjectState status); public void updateWorkflowWithDbId(List members); + public void initializeWorkflow(List members); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java index 58869be54..e6459d11f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java @@ -4,6 +4,7 @@ import org.brapi.v2.model.core.BrAPITrial; import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; @@ -12,9 +13,7 @@ import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; +import java.util.*; import java.util.stream.Collectors; public class PendingTrial implements ExperimentImportEntity { @@ -36,7 +35,12 @@ public List brapiPost(List members) throws ApiException } @Override public List brapiRead() throws ApiException { - return null; + // Get the dbIds of the trials belonging to the required exp units + Set trialDbIds = cache.getObservationUnitByNameNoScope().values().stream() + .map(pendingUnit -> trialService.getTrialDbIdBelongingToPendingUnit(pendingUnit, importContext.getProgram())).collect(Collectors.toSet()); + + // Get the BrAPI trials belonging to required exp units + return trialService.fetchBrapiTrialsByDbId(trialDbIds, importContext.getProgram()); } @Override public List brapiPut(List members) throws ApiException, IllegalArgumentException { @@ -83,4 +87,23 @@ public void updateWorkflowWithDbId(List members) { } } + @Override + public void initializeWorkflow(List members) { + // Check if the input list is of type List + if (!experimentUtilities.isPopulated(members, BrAPITrial.class)) { + return; + } + + // Construct the pending trials from the BrAPI trials + List> pendingTrials = members.stream() + .map(t -> (BrAPITrial) t).map(trialService::constructPIOFromBrapiTrial).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending trial by trial name with the program key removed + Map> pendingTrialByNameNoScope = pendingTrials.stream() + .collect(Collectors.toMap(pio -> Utilities.removeProgramKey(pio.getBrAPIObject().getTrialName(), importContext.getProgram().getKey()), pio -> pio)); + + // Add the map to the context for use in processing import + cache.setTrialByNameNoScope(pendingTrialByNameNoScope); + } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java similarity index 85% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrial.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index 31629f7af..898bbc7f0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -4,10 +4,10 @@ import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPICreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPITrialCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPITrialUpdate; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPITrialCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPITrialUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; @@ -17,7 +17,7 @@ @Slf4j @Prototype @NoArgsConstructor -public class BrAPITrial extends ExpUnitMiddleware { +public class BrAPITrialCommit extends ExpUnitMiddleware { private BrAPITrialCreation brAPITrialCreation; private BrAPITrialUpdate brAPITrialUpdate; private Optional createdBrAPITrials; @@ -34,7 +34,9 @@ public boolean process(ExpUnitMiddlewareContext context) { updatedTrials = brAPITrialUpdate.execute().map(s -> (BrAPIUpdate.BrAPIUpdateState) s); } catch (ApiException e) { - throw new RuntimeException(e); + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); } return processNext(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java index 9deb5f2b0..0ef945d72 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java @@ -6,6 +6,7 @@ import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.BrAPITrialReadWorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; @@ -29,44 +30,22 @@ public RequiredTrials(TrialService trialService) { @Override public boolean process(ExpUnitMiddlewareContext context) { - Program program; - Map> pendingUnitByNameNoScope; - Set trialDbIds; - List brAPITrials; - List> pendingTrials; - Map> pendingTrialByNameNoScope; - - program = context.getImportContext().getProgram(); - pendingUnitByNameNoScope = context.getPendingData().getObservationUnitByNameNoScope(); - - // nothing to do if there are no required units - if (pendingUnitByNameNoScope.size() == 0) { + // Nothing to do if there are no required units + if (context.getPendingData().getObservationUnitByNameNoScope().size() == 0) { return processNext(context); } - log.debug("fetching from BrAPI service, trials belonging to required units"); - // Get the dbIds of the trials belonging to the required exp units - trialDbIds = pendingUnitByNameNoScope.values().stream().map(pendingUnit -> trialService.getTrialDbIdBelongingToPendingUnit(pendingUnit, program)).collect(Collectors.toSet()); try { - // Get the BrAPI trials belonging to required exp units - brAPITrials = trialService.fetchBrapiTrialsByDbId(trialDbIds, program); - - // Construct the pending trials from the BrAPI trials - pendingTrials = brAPITrials.stream().map(trialService::constructPIOFromBrapiTrial).collect(Collectors.toList()); + log.debug("fetching from BrAPI service, trials belonging to required units"); + BrAPITrialReadWorkflowInitialization brAPITrialReadWorkflowInitialization = new BrAPITrialReadWorkflowInitialization(context); + brAPITrialReadWorkflowInitialization.execute(); - // Construct a hashmap to look up the pending trial by trial name with the program key removed - pendingTrialByNameNoScope = pendingTrials.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKey(pio.getBrAPIObject().getTrialName(), program.getKey()), pio -> pio)); - - // Add the map to the context for use in processing import - context.getPendingData().setTrialByNameNoScope(pendingTrialByNameNoScope); } catch (ApiException e) { this.compensate(context, new MiddlewareError(() -> { throw new RuntimeException(e); })); } - - return processNext(context); } } From 15fd790315a83940edd95a8016e3c8d8a9fb5fb4 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 29 May 2024 14:46:40 -0400 Subject: [PATCH 082/203] create PendiningLocation experimentImportEntity --- .../action/create/BrAPICreation.java | 2 +- .../entity/ExperimentImportEntity.java | 4 +- .../entity/PendingLocation.java | 49 +++++++++++++++++++ .../appendoverwrite/entity/PendingTrial.java | 2 +- .../middleware/process/InitialData.java | 2 +- .../middleware/process/OverwrittenData.java | 2 +- .../process/brapi/PendingObservation.java | 2 +- .../DateValidator.java | 2 +- .../FieldValidator.java | 2 +- .../NominalValidator.java | 2 +- .../NumericalValidator.java | 2 +- .../ObservationValidator.java | 2 +- .../OrdinalValidator.java | 2 +- 13 files changed, 61 insertions(+), 14 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{validate => validator}/DateValidator.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{validate => validator}/FieldValidator.java (94%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{validate => validator}/NominalValidator.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{validate => validator}/NumericalValidator.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{validate => validator}/ObservationValidator.java (90%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{validate => validator}/OrdinalValidator.java (97%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java index 6135e188a..3bfed89b5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java @@ -26,7 +26,7 @@ public Optional execute() throws ApiException { List newMembers = entity.copyWorkflowMembers(ImportObjectState.NEW); try { List createdMembers = entity.brapiPost(newMembers); - entity.updateWorkflowWithDbId(createdMembers); + entity.updateWorkflow(createdMembers); return Optional.of(new BrAPICreationState(createdMembers)); } catch (ApiException e) { // TODO: add specific error messages to entity service diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java index daa0e1a70..5105d884d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java @@ -4,8 +4,6 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import java.util.List; -import java.util.function.BiConsumer; -import java.util.function.Function; public interface ExperimentImportEntity { public List brapiPost(List members) throws ApiException; @@ -14,6 +12,6 @@ public interface ExperimentImportEntity { public boolean brapiDelete(List members) throws ApiException; public List getBrAPIState(ImportObjectState status) throws ApiException; public List copyWorkflowMembers(ImportObjectState status); - public void updateWorkflowWithDbId(List members); + public void updateWorkflow(List members); public void initializeWorkflow(List members); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java new file mode 100644 index 000000000..2d54d0e95 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java @@ -0,0 +1,49 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; + +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.model.ProgramLocation; + +import java.util.List; + +public class PendingLocation implements ExperimentImportEntity { + @Override + public List brapiPost(List members) throws ApiException { + return null; + } + + @Override + public List brapiRead() throws ApiException { + return null; + } + + @Override + public List brapiPut(List members) throws ApiException, IllegalArgumentException { + return null; + } + + @Override + public boolean brapiDelete(List members) throws ApiException { + return false; + } + + @Override + public List getBrAPIState(ImportObjectState status) throws ApiException { + return null; + } + + @Override + public List copyWorkflowMembers(ImportObjectState status) { + return null; + } + + @Override + public void updateWorkflow(List members) { + + } + + @Override + public void initializeWorkflow(List members) { + + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java index e6459d11f..cd90b0cb3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java @@ -73,7 +73,7 @@ public List copyWorkflowMembers(ImportObjectState status) { } @Override - public void updateWorkflowWithDbId(List members) { + public void updateWorkflow(List members) { // Check if the input list is of type List if (!experimentUtilities.isPopulated(members, BrAPITrial.class)) { return; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java index 48dd9924f..7a19967e8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java @@ -7,7 +7,7 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java index a6d7c23ee..9a59da97c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java @@ -10,7 +10,7 @@ import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.FieldValidator; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; import org.breedinginsight.utilities.Utilities; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java index e1d2361ce..c618ccc0c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java @@ -24,7 +24,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.*; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/DateValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/DateValidator.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/DateValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/DateValidator.java index 615da28ae..241696289 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/DateValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/DateValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/FieldValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/FieldValidator.java similarity index 94% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/FieldValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/FieldValidator.java index 724aa01d3..de4c7dee4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/FieldValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/FieldValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator; import io.micronaut.context.annotation.Primary; import org.breedinginsight.api.model.v1.response.ValidationError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NominalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/NominalValidator.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NominalValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/NominalValidator.java index 7f2cf50df..2416d6d30 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NominalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/NominalValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NumericalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/NumericalValidator.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NumericalValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/NumericalValidator.java index d90d8d115..97dbd8680 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NumericalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/NumericalValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/ObservationValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/ObservationValidator.java similarity index 90% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/ObservationValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/ObservationValidator.java index e378cd228..0ff664d03 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/ObservationValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/ObservationValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator; import io.micronaut.core.annotation.NonNull; import io.micronaut.core.order.Ordered; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/OrdinalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/OrdinalValidator.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/OrdinalValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/OrdinalValidator.java index 76fbcab04..90c2967c2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/OrdinalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/OrdinalValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; From 402eb5de2dd58f1fa8076db344a88e150a5ed390 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 31 May 2024 10:40:55 -0400 Subject: [PATCH 083/203] create DomainImportService abstract class --- .../model/imports/DomainImportService.java | 72 +++++++++++++++++++ .../ExperimentImportService.java | 32 +++------ .../importer/model/workflow/Workflow.java | 6 ++ 3 files changed, 86 insertions(+), 24 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java new file mode 100644 index 000000000..5f89de64a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -0,0 +1,72 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.imports; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; +import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.List; + +@Singleton +@Slf4j +public abstract class DomainImportService implements BrAPIImportService { + + private final Provider experimentProcessorProvider; + private final Provider processorManagerProvider; + private final Workflow workflowNavigator; + + @Inject + public DomainImportService(Provider experimentProcessorProvider, + Provider processorManagerProvider) + { + this.experimentProcessorProvider = experimentProcessorProvider; + this.processorManagerProvider = processorManagerProvider; + this.workflowNavigator = getNavigator(); + } + + protected abstract Workflow getNavigator(); + @Override + public String getMissingColumnMsg(String columnName) { + return "Column heading does not match template or ontology"; + } + @Override + public List getWorkflows() { + return workflowNavigator.getWorkflows(); + } + + @Override + public ImportPreviewResponse process(ImportServiceContext context) + throws Exception { + + if (!context.getWorkflow().isEmpty()) { + log.info("Workflow: " + context.getWorkflow()); + } + + return workflowNavigator.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); + } +} + diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index c6d191bf0..60fe4bb3b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -19,9 +19,11 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.DomainImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; @@ -33,12 +35,9 @@ @Singleton @Slf4j -public class ExperimentImportService implements BrAPIImportService { +public class ExperimentImportService extends DomainImportService { private final String IMPORT_TYPE_ID = "ExperimentImport"; - - private final Provider experimentProcessorProvider; - private final Provider processorManagerProvider; private final ExperimentWorkflowNavigator workflowNavigator; @Inject @@ -46,10 +45,13 @@ public ExperimentImportService(Provider experimentProcessor Provider processorManagerProvider, ExperimentWorkflowNavigator workflowNavigator) { - this.experimentProcessorProvider = experimentProcessorProvider; - this.processorManagerProvider = processorManagerProvider; + super(experimentProcessorProvider, processorManagerProvider); this.workflowNavigator = workflowNavigator; } + @Override + public Workflow getNavigator() { + return this.workflowNavigator; + } @Override public ExperimentObservation getImportClass() { @@ -61,24 +63,6 @@ public String getImportTypeId() { return IMPORT_TYPE_ID; } - @Override - public String getMissingColumnMsg(String columnName) { - return "Column heading does not match template or ontology"; - } - @Override - public List getWorkflows() { - return workflowNavigator.getWorkflows(); - } - - @Override - public ImportPreviewResponse process(ImportServiceContext context) - throws Exception { - if (!context.getWorkflow().isEmpty()) { - log.info("Workflow: " + context.getWorkflow()); - } - - return workflowNavigator.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); - } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java index 8bc19120e..17f48ee57 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -3,9 +3,15 @@ import io.micronaut.core.order.Ordered; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; @FunctionalInterface public interface Workflow extends Ordered { Optional process(ImportServiceContext context); + default List getWorkflows() { + // Default implementation for getWorkflows method + return new ArrayList<>(); + } } From 69b813f6ead8c24b7ab1b02b85809ae8d8092f26 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 31 May 2024 21:57:03 -0400 Subject: [PATCH 084/203] create pending observation entity --- .../brapi/v2/dao/BrAPIObservationDAO.java | 10 ++ .../create/BrAPIObservationCreation.java | 20 +++ .../entity/ExperimentImportEntity.java | 58 +++++++ .../entity/PendingObservation.java | 152 ++++++++++++++++++ .../appendoverwrite/entity/PendingTrial.java | 1 + ... => BrAPIObservationUpdateMiddleware.java} | 7 +- 6 files changed, 243 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationCreation.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/{BrAPIObservationUpdate.java => BrAPIObservationUpdateMiddleware.java} (90%) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java index 6eb4c2761..6176a9f65 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java @@ -183,6 +183,16 @@ public List getObservationsByStudyName(List studyNames ); } + public List getObservationsByDbIds(List dbIds, Program program) throws ApiException { + if(dbIds.isEmpty()) { + return Collections.emptyList(); + } + + return getProgramObservations(program.getId()).values().stream() + .filter(o -> dbIds.contains(o.getObservationUnitDbId())) + .collect(Collectors.toList()); + } + public List getObservationsByTrialDbId(List trialDbIds, Program program) throws ApiException { if(trialDbIds.isEmpty()) { return Collections.emptyList(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationCreation.java new file mode 100644 index 000000000..74c430fde --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationCreation.java @@ -0,0 +1,20 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +@Slf4j +public class BrAPIObservationCreation extends BrAPICreation { + + public BrAPIObservationCreation(ExpUnitMiddlewareContext context) { + super(context); + } + + @Override + public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + return new PendingObservation(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java index 5105d884d..c7b07837d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java @@ -5,13 +5,71 @@ import java.util.List; +/** + * Interface for importing entities related to experiments using BrAPI service. + */ public interface ExperimentImportEntity { + + /** + * Create new objects generated by the workflow in the BrAPI service. + * @param members List of entities to be created + * @return List of created entities + * @throws ApiException if there is an issue with the API call + */ public List brapiPost(List members) throws ApiException; + + /** + * Fetch objects required by the workflow from the BrAPI service. + * @return List of fetched entities + * @throws ApiException if there is an issue with the API call + */ public List brapiRead() throws ApiException; + + /** + * Commit objects changed by the workflow to the BrAPI service. + * @param members List of entities to be updated + * @param Type of entities + * @return List of updated entities + * @throws ApiException if there is an issue with the API call + * @throws IllegalArgumentException if method arguments are invalid + */ public List brapiPut(List members) throws ApiException, IllegalArgumentException; + + /** + * Remove objects created by the workflow from the BrAPI service. + * @param members List of entities to be deleted + * @param Type of entities + * @return true if deletion is successful, false otherwise + * @throws ApiException if there is an issue with the API call + */ public boolean brapiDelete(List members) throws ApiException; + + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the BrAPI service. + * @param status State of the objects + * @return List of deep copies of entities + * @throws ApiException if there is an issue with the API call + */ public List getBrAPIState(ImportObjectState status) throws ApiException; + + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the workflow context. + * @param status State of the objects + * @return List of deep copies of entities from workflow context + */ public List copyWorkflowMembers(ImportObjectState status); + + /** + * For objects in the workflow context, update any foreign-key fields with values generated by the BrAPI service. + * @param members List of entities to be updated + * @param Type of entities + */ public void updateWorkflow(List members); + + /** + * Populate the workflow context with objects needed by the workflow. + * @param members List of entities to be initialized + * @param Type of entities + */ public void initializeWorkflow(List members); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java new file mode 100644 index 000000000..18201078f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java @@ -0,0 +1,152 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; + +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +public class PendingObservation implements ExperimentImportEntity { + ExpUnitContext cache; + ImportContext importContext; + @Inject + ObservationService observationService; + @Inject + BrAPIObservationDAO brAPIObservationDAO; + @Inject + ExperimentUtilities experimentUtilities; + + public PendingObservation(ExpUnitMiddlewareContext context) { + this.cache = context.getExpUnitContext(); + this.importContext = context.getImportContext(); + } + + + /** + * Create new objects generated by the workflow in the BrAPI service. + * + * @param members List of entities to be created + * @return List of created entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiPost(List members) throws ApiException { + return brAPIObservationDAO.createBrAPIObservations(members, importContext.getProgram().getId(), importContext.getUpload()); + } + + /** + * Fetch objects required by the workflow from the BrAPI service. + * + * @return List of fetched entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiRead() throws ApiException { + return new ArrayList<>(); + } + + /** + * Commit objects changed by the workflow to the BrAPI service. + * + * @param members List of entities to be updated + * @return List of updated entities + * @throws ApiException if there is an issue with the API call + * @throws IllegalArgumentException if method arguments are invalid + */ + @Override + public List brapiPut(List members) throws ApiException, IllegalArgumentException { + // Check if the input list is of type List + if (!experimentUtilities.isPopulated(members, BrAPIObservation.class)) { + return new ArrayList(); + } + + List updatedObservations = new ArrayList<>(); + for (U member : members) { + BrAPIObservation observation = (BrAPIObservation) member; + Optional.ofNullable(brAPIObservationDAO.updateBrAPIObservation(observation.getObservationDbId(), observation, importContext.getProgram().getId())).ifPresent(updatedObservations::add); + } + + return (List) updatedObservations; + } + + /** + * Remove objects created by the workflow from the BrAPI service. + * + * @param members List of entities to be deleted + * @return true if deletion is successful, false otherwise + * @throws ApiException if there is an issue with the API call + */ + @Override + public boolean brapiDelete(List members) throws ApiException { + // TODO: implement delete for observations on BrapiJavaTestServer + return false; + } + + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the BrAPI service. + * + * @param status State of the objects + * @return List of deep copies of entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List getBrAPIState(ImportObjectState status) throws ApiException { + List ids = copyWorkflowMembers(status).stream().map(BrAPIObservation::getObservationDbId).collect(Collectors.toList()); + return brAPIObservationDAO.getObservationsByDbIds(ids, importContext.getProgram()); + } + + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the workflow context. + * + * @param status State of the objects + * @return List of deep copies of entities from workflow context + */ + @Override + public List copyWorkflowMembers(ImportObjectState status) { + return experimentUtilities.copyWorkflowCachePendingBrAPIObjects(cache.getPendingObservationByHash(), BrAPIObservation.class, status); + } + + /** + * For objects in the workflow context, update any foreign-key fields with values generated by the BrAPI service. + * + * @param members List of entities to be updated + */ + @Override + public void updateWorkflow(List members) { + // Check if the input list is of type List + if (!experimentUtilities.isPopulated(members, BrAPIObservation.class)) { + return; + } + + // Update the workflow ref by setting the system-generated dbId for each newly created trial + for (U member : members) { + BrAPIObservation observation = (BrAPIObservation) member; + // TODO:set the observation dBId + } + } + + /** + * Populate the workflow context with objects needed by the workflow. + * + * @param members List of entities to be initialized + */ + @Override + public void initializeWorkflow(List members) { + // Check if the input list is of type List + if (!experimentUtilities.isPopulated(members, BrAPIObservation.class)) { + return; + } + + // TODO:add previous observations + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java index cd90b0cb3..5832b26c2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java @@ -21,6 +21,7 @@ public class PendingTrial implements ExperimentImportEntity { ImportContext importContext; @Inject TrialService trialService; + @Inject BrAPITrialDAO brapiTrialDAO; @Inject ExperimentUtilities experimentUtilities; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUpdateMiddleware.java similarity index 90% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUpdate.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUpdateMiddleware.java index ada8556e5..edb234d2d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUpdateMiddleware.java @@ -4,11 +4,8 @@ import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; -import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; -import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; @@ -20,14 +17,14 @@ @Slf4j @Prototype -public class BrAPIObservationUpdate extends ExpUnitMiddleware { +public class BrAPIObservationUpdateMiddleware extends ExpUnitMiddleware { ExperimentUtilities experimentUtilities; BrAPIObservationDAO brapiObservationDAO; private Map mutatedObservationByDbId; @Inject - public BrAPIObservationUpdate(ExperimentUtilities experimentUtilities, BrAPIObservationDAO brapiObservationDAO) { + public BrAPIObservationUpdateMiddleware(ExperimentUtilities experimentUtilities, BrAPIObservationDAO brapiObservationDAO) { this.experimentUtilities = experimentUtilities; this.brapiObservationDAO = brapiObservationDAO; } From c5dd1390e7eba4dbd06f519939c2c7c6885f43e0 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 3 Jun 2024 11:18:57 -0400 Subject: [PATCH 085/203] Shared services work in progress --- .../experiment/ExperimentUtilities.java | 13 + .../create/model/PendingImportObjectData.java | 36 ++ .../create/model/ProcessContext.java | 31 + .../create/model/ProcessedPhenotypeData.java | 37 ++ .../workflow/CreateNewExperimentWorkflow.java | 6 +- .../CreatePendingImportPopulator.java | 67 ++ ...lateExistingPendingImportObjectsStep.java} | 26 +- ...ulateModifiedPendingImportObjectsStep.java | 5 + .../PopulateNewPendingImportObjectsStep.java | 4 + .../create/workflow/steps/ProcessStep.java | 580 +++++++++++++++++- .../PendingImportObjectPopulator.java | 41 ++ .../services/SharedPendingImportService.java | 22 + .../services/SharedPhenotypeService.java | 76 +++ .../services/SharedSeasonService.java | 97 +++ .../services/SharedValidateService.java | 116 ++++ 15 files changed, 1139 insertions(+), 18 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingImportObjectData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessContext.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/{GetExistingProcessingStep.java => PopulateExistingPendingImportObjectsStep.java} (95%) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateModifiedPendingImportObjectsStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPendingImportService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPhenotypeService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedSeasonService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedValidateService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index f6cb96490..9983b1aaa 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -19,6 +19,8 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.*; import java.util.stream.Collectors; @@ -27,10 +29,21 @@ public class ExperimentUtilities { public static final CharSequence COMMA_DELIMITER = ","; public static final String TIMESTAMP_PREFIX = "TS:"; public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; + public static final String MIDNIGHT = "T00:00:00-00:00"; public static List importRowsToExperimentObservations(List importRows) { return importRows.stream() .map(trialImport -> (ExperimentObservation) trialImport) .collect(Collectors.toList()); } + + public static boolean validDateTimeValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingImportObjectData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingImportObjectData.java new file mode 100644 index 000000000..1f26cdf8d --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingImportObjectData.java @@ -0,0 +1,36 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; + +import lombok.*; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class PendingImportObjectData { + private PendingImportObject trialPIO; + private PendingImportObject studyPIO; + private PendingImportObject obsUnitPIO; + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessContext.java new file mode 100644 index 000000000..b66f7207c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessContext.java @@ -0,0 +1,31 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ProcessContext { + private PendingData pendingData; + private ImportContext importContext; +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java new file mode 100644 index 000000000..cd6842476 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java @@ -0,0 +1,37 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; + +import lombok.*; +import org.breedinginsight.model.Trait; +import tech.tablesaw.columns.Column; + +import java.util.List; +import java.util.Map; + +// TODO: move to common higher level location +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ProcessedPhenotypeData { + private Map> timeStampColByPheno; + private List referencedTraits; + private List> phenotypeCols; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index ebfa5469d..4666ac0f1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -22,7 +22,7 @@ import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.pipeline.Pipeline; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.GetExistingProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.PopulateExistingPendingImportObjectsStep; import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.ProcessStep; import javax.inject.Inject; @@ -38,11 +38,11 @@ @Named("CreateNewExperimentWorkflow") public class CreateNewExperimentWorkflow implements Workflow { - private final Provider getExistingStepProvider; + private final Provider getExistingStepProvider; private final Provider processStepProvider; @Inject - public CreateNewExperimentWorkflow(Provider getExistingStepProvider, + public CreateNewExperimentWorkflow(Provider getExistingStepProvider, Provider processStepProvider) { this.getExistingStepProvider = getExistingStepProvider; this.processStepProvider = processStepProvider; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java new file mode 100644 index 000000000..ea839298b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java @@ -0,0 +1,67 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; + +import org.apache.commons.lang3.StringUtils; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.PendingImportObjectPopulator; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import java.math.BigInteger; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; + +public class CreatePendingImportPopulator implements PendingImportObjectPopulator { + + @Override + public PendingImportObject populateTrial(ImportContext importContext, + PendingData pendingData, + ExperimentObservation importRow, + Supplier expNextVal) + throws UnprocessableEntityException { + + PendingImportObject trialPio; + Program program = importContext.getProgram(); + User user = importContext.getUser(); + boolean commit = importContext.isCommit(); + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + + if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { + PendingImportObject envPio; + trialPio = trialByNameNoScope.get(importRow.getExpTitle()); + envPio = studyByNameNoScope.get(importRow.getEnv()); + + // creating new units for existing experiments and environments is not possible + if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && + (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ + throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); + } + } else if (!trialByNameNoScope.isEmpty()) { + throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); + } else { + UUID id = UUID.randomUUID(); + String expSeqValue = null; + if (commit) { + expSeqValue = expNextVal.get().toString(); + } + BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); + trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); + // TODO: move + //trialByNameNoScope.put(importRow.getExpTitle(), trialPio); + } + + return trialPio; + } + + + + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java similarity index 95% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index 5bba6fd52..8ee355dc3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -40,6 +40,7 @@ import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedStudyService; import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedTrialService; import org.breedinginsight.model.Program; @@ -58,7 +59,7 @@ @Prototype @Slf4j -public class GetExistingProcessingStep implements ProcessingStep { +public class PopulateExistingPendingImportObjectsStep implements ProcessingStep { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final BrAPITrialDAO brAPITrialDAO; @@ -73,14 +74,14 @@ public class GetExistingProcessingStep implements ProcessingStep experimentImportRows = ExperimentUtilities.importRowsToExperimentObservations(input.getImportRows()); Program program = input.getProgram(); @@ -115,7 +116,10 @@ public PendingData process(ImportContext input) { .existingGermplasmByGID(existingGermplasmByGID) .build(); - return existing; + return ProcessContext.builder() + .importContext(input) + .pendingData(existing) + .build(); } /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateModifiedPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateModifiedPendingImportObjectsStep.java new file mode 100644 index 000000000..a04d33540 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateModifiedPendingImportObjectsStep.java @@ -0,0 +1,5 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; + +// TODO: think this would be for other workflow +public class PopulateModifiedPendingImportObjectsStep { +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java new file mode 100644 index 000000000..13870fc51 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -0,0 +1,4 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; + +public class PopulateNewPendingImportObjectsStep { +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java index 8560f6f84..b6f6e7796 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java @@ -16,23 +16,595 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingImportObjectData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.PendingImportObjectPopulator; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedPhenotypeService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedSeasonService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedValidateService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.model.User; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; +import org.jooq.DSLContext; +import tech.tablesaw.api.Table; +import org.breedinginsight.model.Trait; +import tech.tablesaw.columns.Column; + +import javax.inject.Inject; +import java.math.BigInteger; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Collectors; -public class ProcessStep implements ProcessingStep { +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.*; - public ProcessStep() { +@Prototype +@Slf4j +public class ProcessStep implements ProcessingStep { + private final SharedValidateService sharedValidateService; + private final SharedSeasonService sharedSeasonService; + private final SharedPhenotypeService sharedPhenotypeService; + private final BrAPIObservationDAO brAPIObservationDAO; + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private final DSLContext dsl; + + @Inject + public ProcessStep(SharedValidateService sharedValidateService, + SharedSeasonService sharedSeasonService, + SharedPhenotypeService sharedPhenotypeService, + BrAPIObservationDAO brAPIObservationDAO, + BrAPIObservationUnitDAO brAPIObservationUnitDAO, + DSLContext dsl) { + this.sharedValidateService = sharedValidateService; + this.sharedSeasonService = sharedSeasonService; + this.sharedPhenotypeService = sharedPhenotypeService; + this.brAPIObservationDAO = brAPIObservationDAO; + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + this.dsl = dsl; } @Override - public ProcessedData process(PendingData input) { + public ProcessedData process(ProcessContext context) { + + Table data = context.getImportContext().getData(); + ImportUpload upload = context.getImportContext().getUpload(); + ImportContext importContext = context.getImportContext(); + ProcessedPhenotypeData phenotypeData = sharedPhenotypeService.extractPhenotypes(importContext); // TODO: implement return new ProcessedData(); } + + +/* + // initNew + private void populatePendingImportObjects(ImportContext importContext, + ProcessedPhenotypeData phenotypeData, + PendingImportObjectPopulator pioPopulator) { + + List importRows = importContext.getImportRows(); + Program program = importContext.getProgram(); + + Supplier expNextVal = getNextExperimentSequenceNumber(program); + Supplier envNextVal = getNextEnvironmentSequenceNumber(program); + + // TODO: handle this + // existingObsByObsHash = fetchExistingObservations(referencedTraits, program); + + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + + populateIndependentVariablePIOsForRow(); + + // ... (Common logic from the original method) + + PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); + + processObservations(importContext, phenotypeData, importRow, rowNum, commit, importRow.getEnvYear(), obsUnitPIO, studyPIO); + } + } +*/ + + /** + * Returns a Supplier that generates the next experiment sequence number based on the given Program. + * + * @param program the Program for which to generate the next experiment sequence number + * @return a Supplier that generates the next experiment sequence number + * @throws HttpStatusException if the program is not properly configured for observation unit import + */ + private Supplier getNextExperimentSequenceNumber(Program program) { + String expSequenceName = program.getExpSequence(); + if (expSequenceName == null) { + log.error(String.format("Program, %s, is missing a value in the exp sequence column.", program.getName())); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for observation unit import"); + } + return () -> dsl.nextval(expSequenceName.toLowerCase()); + } + + /** + * Retrieves the next environment sequence number for a given program. + * + * @param program The program for which to get the next environment sequence number. + * @return A Supplier representing a function that generates the next environment sequence number. + * @throws HttpStatusException If the program is not properly configured for environment import. + */ + private Supplier getNextEnvironmentSequenceNumber(Program program) { + String envSequenceName = program.getEnvSequence(); + if (envSequenceName == null) { + log.error(String.format("Program, %s, is missing a value in the env sequence column.", program.getName())); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for environment import"); + } + return () -> dsl.nextval(envSequenceName.toLowerCase()); + } + + /** + * Populates independent variable PendingImportObjectData for a given row of import data. + * + * @param importContext The import context. + * @param phenotypeData The processed phenotype data. + * @param pendingData The pending data. + * @param importRow The import row. + * @param expNextVal The supplier for generating experiment next value. + * @param envNextVal The supplier for generating environment next value. + * @param pioPopulator The pending import object populator. + * @return The populated independent variable PendingImportObjectData. + * @throws MissingRequiredInfoException If any required information is missing. + * @throws UnprocessableEntityException If the entity is unprocessable. + * @throws ApiException If there is an API exception. + */ + private PendingImportObjectData populateIndependentVariablePIOsForRow(ImportContext importContext, + ProcessedPhenotypeData phenotypeData, + PendingData pendingData, + ExperimentObservation importRow, + Supplier expNextVal, + Supplier envNextVal, + PendingImportObjectPopulator pioPopulator) + throws MissingRequiredInfoException, UnprocessableEntityException, ApiException { + + Program program = importContext.getProgram(); + User user = importContext.getUser(); + boolean commit = importContext.isCommit(); + List referencedTraits = phenotypeData.getReferencedTraits(); + + PendingImportObject trialPIO = null; + try { + trialPIO = pioPopulator.populateTrial(importContext, importRow, expNextVal); + + // moved up a level + if (trialPIO.getState() == ImportObjectState.NEW) { + pendingData.getTrialByNameNoScope().put(importRow.getExpTitle(), trialPIO); + } + } catch (UnprocessableEntityException e) { + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } + + String expSeqValue = null; + if (commit) { + expSeqValue = trialPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString(); + } + + if (commit) { + fetchOrCreateDatasetPIO(importRow, program, referencedTraits); + } + + fetchOrCreateLocationPIO(importRow); + + PendingImportObject studyPIO = fetchOrCreateStudyPIO(program, commit, expSeqValue, importRow, envNextVal); + + String envSeqValue = null; + if (commit) { + envSeqValue = studyPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER) + .getAsString(); + } + + PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); + + return PendingImportObjectData.builder() + .trialPIO(trialPIO) + .studyPIO(studyPIO) + .obsUnitPIO(obsUnitPIO) + .build(); + } + +/* + private void processObservations(ImportContext importContext, ProcessedPhenotypeData phenotypeData, ExperimentObservation importRow, + int rowNum, boolean commit, String studyYear, + PendingImportObject obsUnitPIO, + PendingImportObject studyPIO) + throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { + Program program = importContext.getProgram(); + User user = importContext.getUser(); + List> phenotypeCols = phenotypeData.getPhenotypeCols(); + Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); + List referencedTraits = phenotypeData.getReferencedTraits(); + + for (Column column : phenotypeCols) { + // ... (Logic for processing observations) + } + } + + // TODO: move common code out + private void initNewBrapiData(ImportContext importContext, ProcessedPhenotypeData phenotypeData) + throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { + + Program program = importContext.getProgram(); + List importRows = importContext.getImportRows(); + User user = importContext.getUser(); + boolean commit = importContext.isCommit(); + Map existingObsByObsHash; + + List referencedTraits = phenotypeData.getReferencedTraits(); + List> phenotypeCols = phenotypeData.getPhenotypeCols(); + Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); + + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + + + for (Column column : phenotypeCols) { + //If associated timestamp column, add + String dateTimeValue = null; + if (timeStampColByPheno.containsKey(column.name())) { + dateTimeValue = timeStampColByPheno.get(column.name()).getString(rowNum); + //If no timestamp, set to midnight + if (!dateTimeValue.isBlank() && !validDateTimeValue(dateTimeValue)) { + dateTimeValue += MIDNIGHT; + } + } + + // get the study year either referenced from the observation unit or listed explicitly on the import row + // TODO: handle this different workflows + String studyYear = hasAllReferenceUnitIds ? studyPIO.getBrAPIObject().getSeasons().get(0) : importRow.getEnvYear(); + String seasonDbId = sharedSeasonService.yearToSeasonDbId(studyYear, program.getId()); + fetchOrCreateObservationPIO( + program, + user, + importRow, + column, //column.name() gets phenotype name + rowNum, + dateTimeValue, + commit, + seasonDbId, + obsUnitPIO, + studyPIO, + referencedTraits + ); + } + } + } + + private Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { + Set ouDbIds = new HashSet<>(); + Set variableDbIds = new HashSet<>(); + Map variableNameByDbId = new HashMap<>(); + Map ouNameByDbId = new HashMap<>(); + Map studyNameByDbId = studyByNameNoScope.values() + .stream() + .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) + .map(PendingImportObject::getBrAPIObject) + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); + + studyNameByDbId.keySet().forEach(studyDbId -> { + try { + brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { + if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { + ouDbIds.add(ou.getObservationUnitDbId()); + } + ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + }); + } catch (ApiException e) { + throw new RuntimeException(e); + } + }); + + for (Trait referencedTrait : referencedTraits) { + variableDbIds.add(referencedTrait.getObservationVariableDbId()); + variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); + } + + List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); + + return existingObservations.stream() + .map(obs -> { + String studyName = studyNameByDbId.get(obs.getStudyDbId()); + String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); + String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); + + String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); + + return Map.entry(key, obs); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private PendingImportObject fetchOrCreateTrialPIO( + Program program, + User user, + boolean commit, + ExperimentObservation importRow, + Supplier expNextVal + ) throws UnprocessableEntityException { + PendingImportObject trialPio; + + // use the prior trial if observation unit IDs are supplied + // TODO: handle multiple workflows + if (hasAllReferenceUnitIds) { + trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); + + // otherwise create a new trial, but there can be only one allowed + } else { + if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { + PendingImportObject envPio; + trialPio = trialByNameNoScope.get(importRow.getExpTitle()); + envPio = studyByNameNoScope.get(importRow.getEnv()); + + // creating new units for existing experiments and environments is not possible + if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && + (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ + throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); + } + } else if (!trialByNameNoScope.isEmpty()) { + throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); + } else { + UUID id = UUID.randomUUID(); + String expSeqValue = null; + if (commit) { + expSeqValue = expNextVal.get().toString(); + } + BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); + trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); + trialByNameNoScope.put(importRow.getExpTitle(), trialPio); + } + } + return trialPio; + } + + private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) throws UnprocessableEntityException { + PendingImportObject pio; + + // TODO: multiple workflows + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + String name = String.format("Observation Dataset [%s-%s]", + program.getKey(), + trialPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString()); + if (obsVarDatasetByName.containsKey(name)) { + pio = obsVarDatasetByName.get(name); + } else { + UUID id = UUID.randomUUID(); + BrAPIListDetails newDataset = importRow.constructDatasetDetails( + name, + id, + BRAPI_REFERENCE_SOURCE, + program, + trialPIO.getId().toString()); + pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); + trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); + if (ImportObjectState.EXISTING == trialPIO.getState()) { + trialPIO.setState(ImportObjectState.MUTATED); + } + obsVarDatasetByName.put(name, pio); + } + addObsVarsToDatasetDetails(pio, referencedTraits, program); + } + + private void fetchOrCreateLocationPIO(ExperimentObservation importRow) { + PendingImportObject pio; + // TODO: multiple workflows + String envLocationName = hasAllReferenceUnitIds ? + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName() : importRow.getEnvLocation(); + if (!locationByName.containsKey((importRow.getEnvLocation()))) { + ProgramLocation newLocation = new ProgramLocation(); + newLocation.setName(envLocationName); + pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); + this.locationByName.put(envLocationName, pio); + } + } + + private PendingImportObject fetchOrCreateStudyPIO( + Program program, + boolean commit, + String expSequenceValue, + ExperimentObservation importRow, + Supplier envNextVal + ) throws UnprocessableEntityException { + PendingImportObject pio; + // TODO: multiple workflows + if (hasAllReferenceUnitIds) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), + program.getKey() + ); + pio = studyByNameNoScope.get(studyName); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { + pio = studyByNameNoScope.get(importRow.getEnv()); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else { + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + UUID trialID = trialPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); + newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies + + // It is assumed that the study has only one season, And that the Years and not + // the dbId's are stored in getSeason() list. + String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season + if (commit) { + if(StringUtils.isNotBlank(year)) { + String seasonID = this.yearToSeasonDbId(year, program.getId()); + newStudy.setSeasons(Collections.singletonList(seasonID)); + } + } else { + addYearToStudyAdditionalInfo(program, newStudy, year); + } + + pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); + this.studyByNameNoScope.put(importRow.getEnv(), pio); + } + return pio; + } + + private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { + PendingImportObject pio; + String key = createObservationUnitKey(importRow); + // TODO: multiple workflows + if (hasAllReferenceUnitIds) { + pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); + } else if (observationUnitByNameNoScope.containsKey(key)) { + pio = observationUnitByNameNoScope.get(key); + } else { + String germplasmName = ""; + if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { + germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) + .getBrAPIObject() + .getGermplasmName(); + } + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; + UUID trialID = trialPIO.getId(); + UUID datasetId = null; + if (commit) { + datasetId = UUID.fromString(trialPIO.getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + } + PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); + + // check for existing units if this is an existing study + if (studyPIO.getBrAPIObject().getStudyDbId() != null) { + List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); + List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); + if (matchingOU.isEmpty()) { + throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); + } else { + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); + } + } else { + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); + } + this.observationUnitByNameNoScope.put(key, pio); + } + return pio; + } + + private void fetchOrCreateObservationPIO(Program program, + User user, + ExperimentObservation importRow, + Column column, + Integer rowNum, + String timeStampValue, + boolean commit, + String seasonDbId, + PendingImportObject obsUnitPIO, + PendingImportObject studyPIO, + List referencedTraits) throws ApiException, UnprocessableEntityException { + PendingImportObject pio; + BrAPIObservation newObservation; + String variableName = column.name(); + String value = column.getString(rowNum); + String key; + + // TODO: multiple workflows + if (hasAllReferenceUnitIds) { + String unitName = obsUnitPIO.getBrAPIObject().getObservationUnitName(); + String studyName = studyPIO.getBrAPIObject().getStudyName(); + key = getObservationHash(studyName + unitName, variableName, studyName); + } else { + key = getImportObservationHash(importRow, variableName); + } + + if (existingObsByObsHash.containsKey(key)) { + if (!isObservationMatched(key, value, column, rowNum)){ + + // prior observation with updated value + newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); + if (!isValueMatched(key, value)){ + newObservation.setValue(value); + } else if (!StringUtils.isBlank(timeStampValue) && !isTimestampMatched(key, timeStampValue)) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timeStampValue)); + newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + } + pio = new PendingImportObject<>(ImportObjectState.MUTATED, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); + } else { + + // prior observation + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(existingObsByObsHash.get(key), BrAPIObservation.class, program)); + } + + observationByHash.put(key, pio); + } else if (!this.observationByHash.containsKey(key)){ + + // new observation + // TODO: multiple workflows + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + + UUID trialID = trialPIO.getId(); + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject(), commit, program, user, BRAPI_REFERENCE_SOURCE, trialID, studyID, obsUnitPIO.getId(), id); + //NOTE: Can't parse invalid timestamp value, so have to skip if invalid. + // Validation error should be thrown for offending value, but that doesn't happen until later downstream + if (timeStampValue != null && !timeStampValue.isBlank() && (validDateValue(timeStampValue) || validDateTimeValue(timeStampValue))) { + newObservation.setObservationTimeStamp(OffsetDateTime.parse(timeStampValue)); + } + + newObservation.setStudyDbId(studyPIO.getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations + + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); + this.observationByHash.put(key, pio); + } + } + + */ + } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java new file mode 100644 index 000000000..d189a461d --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java @@ -0,0 +1,41 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import java.math.BigInteger; +import java.util.function.Supplier; + +public interface PendingImportObjectPopulator { + + PendingImportObject populateTrial( + ImportContext importContext, + PendingData pendingData, + ExperimentObservation importRow, + Supplier expNextVal + ) throws UnprocessableEntityException; + + + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPendingImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPendingImportService.java new file mode 100644 index 000000000..a5d091d69 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPendingImportService.java @@ -0,0 +1,22 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +public class SharedPendingImportService { + + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPhenotypeService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPhenotypeService.java new file mode 100644 index 000000000..6b9bd7dc5 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPhenotypeService.java @@ -0,0 +1,76 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.DynamicColumnParser; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import tech.tablesaw.api.Table; +import tech.tablesaw.columns.Column; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.TIMESTAMP_REGEX; + +@Singleton +@Slf4j +public class SharedPhenotypeService { + + private final SharedValidateService sharedValidateService; + + @Inject + public SharedPhenotypeService(SharedValidateService sharedValidateService) { + this.sharedValidateService = sharedValidateService; + } + + /** + * Extracts phenotypes from the import context. + * + * @param importContext The import context containing the data, upload, and program information. + * @return A ProcessedPhenotypeData object with the extracted phenotypes. + */ + public ProcessedPhenotypeData extractPhenotypes(ImportContext importContext) { + Table data = importContext.getData(); + ImportUpload upload = importContext.getUpload(); + Program program = importContext.getProgram(); + + DynamicColumnParser.DynamicColumnParseResult result = DynamicColumnParser.parse(data, upload.getDynamicColumnNames()); + List traits = sharedValidateService.verifyTraits(program.getId(), result); + + Map> timeStampColByPheno = new HashMap<>(); + //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval + for (Column tsColumn : result.getTimestampCols()) { + timeStampColByPheno.put(tsColumn.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), tsColumn); + } + + return ProcessedPhenotypeData.builder() + .referencedTraits(traits) + .phenotypeCols(result.getPhenotypeCols()) + .timeStampColByPheno(timeStampColByPheno) + .build(); + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedSeasonService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedSeasonService.java new file mode 100644 index 000000000..d168a8a16 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedSeasonService.java @@ -0,0 +1,97 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPISeason; +import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@Singleton +@Slf4j +public class SharedSeasonService { + + private final BrAPISeasonDAO brAPISeasonDAO; + + // TODO: move season to actual cache rather than cacheing at application layer + private Map yearToSeasonDbIdCache = new HashMap<>(); + + @Inject + public SharedSeasonService(BrAPISeasonDAO brAPISeasonDAO) { + this.brAPISeasonDAO = brAPISeasonDAO; + } + + /** + * Converts year String to SeasonDbId + *
+ * NOTE: This assumes that the only Season records of interest are ones + * with a blank name or a name that is the same as the year. + * + * @param year The year as a string + * @param programId the program ID. + * @return the DbId of the season-record associated with the year + */ + public String yearToSeasonDbId(String year, UUID programId) { + String dbID = null; + if (yearToSeasonDbIdCache.containsKey(year)) { // get it from cache if possible + dbID = yearToSeasonDbIdCache.get(year); + } else { + dbID = yearToSeasonDbIdFromDatabase(year, programId); + yearToSeasonDbIdCache.put(year, dbID); + } + return dbID; + } + + private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { + BrAPISeason targetSeason = null; + List seasons; + try { + seasons = brAPISeasonDAO.getSeasonsByYear(year, programId); + for (BrAPISeason season : seasons) { + if (null == season.getSeasonName() || season.getSeasonName().isBlank() || season.getSeasonName().equals(year)) { + targetSeason = season; + break; + } + } + if (targetSeason == null) { + BrAPISeason newSeason = new BrAPISeason(); + Integer intYear = null; + if( StringUtils.isNotBlank(year) ){ + intYear = Integer.parseInt(year); + } + newSeason.setYear(intYear); + newSeason.setSeasonName(year); + targetSeason = brAPISeasonDAO.addOneSeason(newSeason, programId); + } + + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + log.error(e.getResponseBody(), e); + } + + return (targetSeason == null) ? null : targetSeason.getSeasonDbId(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedValidateService.java new file mode 100644 index 000000000..f2edfc3f5 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedValidateService.java @@ -0,0 +1,116 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.brapps.importer.services.FileMappingUtil; +import org.breedinginsight.brapps.importer.services.processors.experiment.DynamicColumnParser.DynamicColumnParseResult; +import org.breedinginsight.dao.db.tables.pojos.TraitEntity; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import tech.tablesaw.columns.Column; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.COMMA_DELIMITER; +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.TIMESTAMP_REGEX; + +@Singleton +@Slf4j +public class SharedValidateService { + + private final OntologyService ontologyService; + private final FileMappingUtil fileMappingUtil; + + @Inject + public SharedValidateService(OntologyService ontologyService, FileMappingUtil fileMappingUtil) { + this.ontologyService = ontologyService; + this.fileMappingUtil = fileMappingUtil; + } + + /** + * Verifies traits based on program ID and dynamic column parse result. + * + * @param programId The UUID of the program. + * @param cols The dynamic column parse result object containing phenotype and timestamp columns. + * @return The list of verified traits. + * @throws HttpStatusException If ontology terms are not found or timestamp columns lack corresponding phenotype columns. + */ + public List verifyTraits(UUID programId, DynamicColumnParseResult cols) { + Set varNames = cols.getPhenotypeCols().stream() + .map(Column::name) + .collect(Collectors.toSet()); + Set tsNames = cols.getTimestampCols().stream() + .map(Column::name) + .collect(Collectors.toSet()); + + // filter out just traits specified in file + List filteredTraits = fetchFileTraits(programId, varNames); + + // check that all specified ontology terms were found + if (filteredTraits.size() != varNames.size()) { + Set returnedVarNames = filteredTraits.stream() + .map(TraitEntity::getObservationVariableName) + .collect(Collectors.toSet()); + List differences = varNames.stream() + .filter(var -> !returnedVarNames.contains(var)) + .collect(Collectors.toList()); + //TODO convert this to a ValidationError + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, + "Ontology term(s) not found: " + String.join(COMMA_DELIMITER, differences)); + } + + // Check that each ts column corresponds to a phenotype column + List unmatchedTimestamps = tsNames.stream() + .filter(e -> !(varNames.contains(e.replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY)))) + .collect(Collectors.toList()); + if (unmatchedTimestamps.size() > 0) { + //TODO convert this to a ValidationError + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, + "Timestamp column(s) lack corresponding phenotype column(s): " + String.join(COMMA_DELIMITER, unmatchedTimestamps)); + } + + // sort the verified traits to match the order of the trait columns + List phenotypeColNames = cols.getPhenotypeCols().stream().map(Column::name).collect(Collectors.toList()); + return fileMappingUtil.sortByField(phenotypeColNames, filteredTraits, TraitEntity::getObservationVariableName); + } + + private List fetchFileTraits(UUID programId, Collection varNames) { + try { + Collection upperCaseVarNames = varNames.stream().map(String::toUpperCase).collect(Collectors.toList()); + List traits = ontologyService.getTraitsByProgramId(programId, true); + // filter out just traits specified in file + return traits.stream() + .filter(e -> upperCaseVarNames.contains(e.getObservationVariableName().toUpperCase())) + .collect(Collectors.toList()); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + throw new InternalServerException(e.toString(), e); + } + } +} From c87a355a5694eabfedc6a645501d404c902b7168 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 4 Jun 2024 10:15:10 -0400 Subject: [PATCH 086/203] update workflow pending trial --- .../entity/ExperimentImportEntity.java | 2 +- .../entity/PendingObservation.java | 2 +- .../appendoverwrite/entity/PendingTrial.java | 70 ++++++++++++++++++- 3 files changed, 71 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java index c7b07837d..08ef5ff82 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java @@ -53,7 +53,7 @@ public interface ExperimentImportEntity { public List getBrAPIState(ImportObjectState status) throws ApiException; /** - * For workflow pending import objects of a given state, fetch deep copies of the objects from the workflow context. + * For workflow pending import objects of a given state, construct deep copies of the objects from the workflow context. * @param status State of the objects * @return List of deep copies of entities from workflow context */ diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java index 18201078f..3854a9d9a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java @@ -106,7 +106,7 @@ public List getBrAPIState(ImportObjectState status) throws Api } /** - * For workflow pending import objects of a given state, fetch deep copies of the objects from the workflow context. + * For workflow pending import objects of a given state, construct deep copies of the objects from the workflow context. * * @param status State of the objects * @return List of deep copies of entities from workflow context diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java index 5832b26c2..cd5e3c6bc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java @@ -30,10 +30,25 @@ public PendingTrial(ExpUnitMiddlewareContext context) { this.cache = context.getExpUnitContext(); this.importContext = context.getImportContext(); } + + /** + * Create new objects generated by the workflow in the BrAPI service. + * + * @param members List of entities to be created + * @return List of created entities + * @throws ApiException if there is an issue with the API call + */ @Override public List brapiPost(List members) throws ApiException { return brapiTrialDAO.createBrAPITrials(members, importContext.getProgram().getId(), importContext.getUpload()); } + + /** + * Fetch objects required by the workflow from the BrAPI service. + * + * @return List of fetched entities + * @throws ApiException if there is an issue with the API call + */ @Override public List brapiRead() throws ApiException { // Get the dbIds of the trials belonging to the required exp units @@ -43,6 +58,15 @@ public List brapiRead() throws ApiException { // Get the BrAPI trials belonging to required exp units return trialService.fetchBrapiTrialsByDbId(trialDbIds, importContext.getProgram()); } + + /** + * Commit objects changed by the workflow to the BrAPI service. + * + * @param members List of entities to be updated + * @return List of updated entities + * @throws ApiException if there is an issue with the API call + * @throws IllegalArgumentException if method arguments are invalid + */ @Override public List brapiPut(List members) throws ApiException, IllegalArgumentException { // Check if the input list is of type List @@ -58,21 +82,49 @@ public List brapiPut(List members) throws ApiException, IllegalArgumen return (List) updatedTrials; } + + /** + * Remove objects created by the workflow from the BrAPI service. + * + * @param members List of entities to be deleted + * @return true if deletion is successful, false otherwise + * @throws ApiException if there is an issue with the API call + */ @Override public boolean brapiDelete(List members) throws ApiException { // TODO: implement delete for trials on BrapiJavaTestServer return false; } + + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the BrAPI service. + * + * @param status State of the objects + * @return List of deep copies of entities + * @throws ApiException if there is an issue with the API call + */ @Override public List getBrAPIState(ImportObjectState status) throws ApiException { List ids = copyWorkflowMembers(status).stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList()); return brapiTrialDAO.getTrialsByDbIds(ids, importContext.getProgram()); } + + /** + * For workflow pending import objects of a given state, construct deep copies of the objects from the workflow context. + * + * @param status State of the objects + * @return List of deep copies of entities from workflow context + */ @Override public List copyWorkflowMembers(ImportObjectState status) { return experimentUtilities.copyWorkflowCachePendingBrAPIObjects(cache.getTrialByNameNoScope(), BrAPITrial.class, status); } + /** + * For objects in the workflow context, update any foreign-key fields with values generated by the BrAPI service. + * + * @param members List of entities to be updated + */ @Override public void updateWorkflow(List members) { // Check if the input list is of type List @@ -86,8 +138,25 @@ public void updateWorkflow(List members) { String createdTrialNameNoScope = Utilities.removeProgramKey(trial.getTrialName(), importContext.getProgram().getKey()); cache.getTrialByNameNoScope().get(createdTrialNameNoScope).getBrAPIObject().setTrialDbId(trial.getTrialDbId()); } + + // update trial DbIds in studies for all distinct trials + cache.getTrialByNameNoScope().values().stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(trial -> + cache.getStudyByNameNoScope().values().stream() + .filter(study -> study.getBrAPIObject().getTrialName() + .equals(Utilities.removeProgramKey(trial.getTrialName(), importContext.getProgram().getKey()))) + .forEach(study -> study.getBrAPIObject().setTrialDbId(trial.getTrialDbId())) + ); } + /** + * Populate the workflow context with objects needed by the workflow. + * + * @param members List of entities to be initialized + */ @Override public void initializeWorkflow(List members) { // Check if the input list is of type List @@ -106,5 +175,4 @@ public void initializeWorkflow(List members) { // Add the map to the context for use in processing import cache.setTrialByNameNoScope(pendingTrialByNameNoScope); } - } From bfc327309fef55aa43593b7d64381b70d653d1bd Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:25:49 -0400 Subject: [PATCH 087/203] create pending location and location creation --- .../appendoverwrite/action/BrAPIAction.java | 34 +++++ .../action/create/BrAPICreation.java | 41 +++++- .../action/create/LocationCreation.java | 28 ++++ .../read/BrAPIReadWorkflowInitialization.java | 35 ++++- .../LocationReadWorkflowInitialization.java | 31 ++++ .../entity/ExperimentImportEntity.java | 5 +- .../entity/PendingLocation.java | 137 +++++++++++++++++- .../appendoverwrite/entity/PendingTrial.java | 2 +- .../middleware/ExpUnitMiddleware.java | 11 ++ ...a => BrAPILocationCreationMiddleware.java} | 19 +-- .../middleware/commit/LocationCommit.java | 41 ++++++ 11 files changed, 356 insertions(+), 28 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/LocationCreation.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/{BrAPILocationCreation.java => BrAPILocationCreationMiddleware.java} (75%) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIAction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIAction.java index 201139a5f..318337df3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIAction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIAction.java @@ -6,7 +6,41 @@ import java.util.Optional; +/** + * Interface representing an action to be performed on the BrAPI service. + * This interface defines two methods: execute() to execute the action on the BrAPI service + * and return any relevant BrAPI state, and getEntity() to get the BrAPI entity being + * acted on based on the provided ExpUnitMiddlewareContext. + * + * @param The type of entity on which the action is being performed. + */ public interface BrAPIAction { + + /** + * Execute the action on the BrAPI service. + * + * @return An Optional containing the relevant BrAPI state after executing the action. + * @throws ApiException if an error occurs during the execution of the action. + */ Optional execute() throws ApiException; + + /** + * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. + * + * @param context The ExpUnitMiddlewareContext providing information about the entity. + * @return The ExperimentImportEntity representing the BrAPI entity being acted on. + */ ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context); } + + +/* + * Overall module description: + * This BrAPIAction interface defines methods to handle actions on the BrAPI service. + * Developers can implement this interface to execute actions and retrieve the entity being acted upon. + * The execute() method is used to perform actions on the BrAPI service and return the resulting BrAPI state, + * while the getEntity() method retrieves the BrAPI entity based on the provided ExpUnitMiddlewareContext. + * + * The BrAPIAction interface allows for customization of BrAPI service interactions and handling of BrAPI entities. + */ + diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java index 3bfed89b5..063781a79 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java @@ -9,47 +9,80 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import java.util.List; import java.util.Optional; +/** + * The BrAPICreation class is an abstract class that implements the BrAPIAction interface. It is responsible for creating entities in a BrAPI system. + * + * @param the type of entity being created + */ @Slf4j public abstract class BrAPICreation implements BrAPIAction { + ExperimentImportEntity entity; + /** + * Constructor for BrAPICreation class. + * @param context the ExpUnitMiddlewareContext object + */ protected BrAPICreation(ExpUnitMiddlewareContext context) { this.entity = getEntity(context); } + /** + * Abstract method to get the ExperimentImportEntity based on the ExpUnitMiddlewareContext. + * @param context the ExpUnitMiddlewareContext object + * @return the ExperimentImportEntity object + */ public abstract ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context); + /** + * Executes the creation process for entities. + * @return an Optional containing the BrAPI state after execution + * @throws ApiException if an error occurs during execution + */ public Optional execute() throws ApiException { List newMembers = entity.copyWorkflowMembers(ImportObjectState.NEW); try { List createdMembers = entity.brapiPost(newMembers); entity.updateWorkflow(createdMembers); - return Optional.of(new BrAPICreationState(createdMembers)); - } catch (ApiException e) { - // TODO: add specific error messages to entity service + return Optional.of(new BrAPICreationState<>(createdMembers)); + } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { log.error("Error creating..."); throw new InternalServerException("Error creating...", e); } } + /** + * Inner class representing the state of creation for BrAPI entities. + * @param the type of entity + */ @Getter public class BrAPICreationState implements BrAPIState { private final List members; + /** + * Constructor for BrAPICreationState class. + * @param createdMembers the list of created members + */ public BrAPICreationState(List createdMembers) { this.members = createdMembers; } + /** + * Undo the creation operation by deleting the created members. + * @return true if undo operation is successful, false otherwise + */ public boolean undo() { List createdMembers = this.getMembers(); try { return entity.brapiDelete(createdMembers); } catch (ApiException e) { - // TODO: add specific error messages to entity service log.error("Error deleting..."); throw new InternalServerException("Error deleting...", e); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/LocationCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/LocationCreation.java new file mode 100644 index 000000000..a3def6c0c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/LocationCreation.java @@ -0,0 +1,28 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; + +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingLocation; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.model.ProgramLocation; + +public class LocationCreation extends BrAPICreation{ + /** + * Constructor for BrAPICreation class. + * + * @param context the ExpUnitMiddlewareContext object + */ + public LocationCreation(ExpUnitMiddlewareContext context) { + super(context); + } + + /** + * Abstract method to get the ExperimentImportEntity based on the ExpUnitMiddlewareContext. + * + * @param context the ExpUnitMiddlewareContext object + * @return the ExperimentImportEntity object + */ + @Override + public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + return new PendingLocation(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java index 364049996..f2aa5bfef 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java @@ -12,14 +12,33 @@ import java.util.List; import java.util.Optional; +/** + * This abstract class, BrAPIReadWorkflowInitialization, is responsible for initializing and executing a read workflow for a given BrAPI entity. + * It implements the BrAPIAction interface and uses the ExperimentImportEntity to perform read operations. + * + * @param the type of entity for which the workflow is initialized + */ @Slf4j public abstract class BrAPIReadWorkflowInitialization implements BrAPIAction { - ExperimentImportEntity entity; + ExperimentImportEntity entity; // The entity used for read operations initialization + + /** + * Constructs a new BrAPIReadWorkflowInitialization object with the given ExpUnitMiddlewareContext. + * Initializes the entity based on the provided context. + * + * @param context the ExpUnitMiddlewareContext used for initialization. + */ protected BrAPIReadWorkflowInitialization(ExpUnitMiddlewareContext context) { this.entity = getEntity(context); } + /** + * Executes the read workflow by fetching members from the entity and initializing the workflow. + * + * @return an Optional containing the BrAPIState representing the completed read workflow + * @throws ApiException if an error occurs during execution + */ public Optional execute() throws ApiException { try { List fetchedMembers = entity.brapiRead(); @@ -31,10 +50,22 @@ public Optional execute() throws ApiException { throw new InternalServerException("Error reading...", e); } } + + /** + * The state class representing the result of a read operation. + * + * @param the type of entity members contained in the state + */ @Getter public static class BrAPIReadState implements BrAPIState { - private final List members; + private final List members; // The list of members fetched during the read operation + + /** + * Constructs a new BrAPIReadState object with the provided list of members. + * + * @param fetchedMembers the list of members fetched during the read operation + */ public BrAPIReadState(List fetchedMembers) { this.members = fetchedMembers; } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java new file mode 100644 index 000000000..ba563692e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java @@ -0,0 +1,31 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingLocation; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.model.ProgramLocation; + +@Slf4j +public class LocationReadWorkflowInitialization extends BrAPIReadWorkflowInitialization{ + /** + * Constructs a new BrAPIReadWorkflowInitialization object with the given ExpUnitMiddlewareContext. + * Initializes the entity based on the provided context. + * + * @param context the ExpUnitMiddlewareContext used for initialization. + */ + protected LocationReadWorkflowInitialization(ExpUnitMiddlewareContext context) { + super(context); + } + + /** + * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. + * + * @param context The ExpUnitMiddlewareContext providing information about the entity. + * @return The ExperimentImportEntity representing the BrAPI entity being acted on. + */ + @Override + public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + return new PendingLocation(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java index 08ef5ff82..6edf2afc6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java @@ -2,6 +2,9 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import java.util.List; @@ -16,7 +19,7 @@ public interface ExperimentImportEntity { * @return List of created entities * @throws ApiException if there is an issue with the API call */ - public List brapiPost(List members) throws ApiException; + public List brapiPost(List members) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException; /** * Fetch objects required by the workflow from the BrAPI service. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java index 2d54d0e95..8e2f02269 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java @@ -1,49 +1,174 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.ProgramLocationService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import javax.inject.Inject; +import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; public class PendingLocation implements ExperimentImportEntity { + ExpUnitContext cache; + ImportContext importContext; + @Inject + ProgramLocationService programLocationService; + @Inject + LocationService locationService; + @Inject + ExperimentUtilities experimentUtilities; + public PendingLocation(ExpUnitMiddlewareContext context) { + this.cache = context.getExpUnitContext(); + this.importContext = context.getImportContext(); + } + + /** + * Create new objects generated by the workflow in the BrAPI service. + * + * @param members List of entities to be created + * @return List of created entities + * @throws ApiException if there is an issue with the API call + */ @Override - public List brapiPost(List members) throws ApiException { - return null; + public List brapiPost(List members) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + // Construct requests + List locationRequests = members.stream() + .map(location -> ProgramLocationRequest.builder() + .name(location.getName()) + .build()) + .collect(Collectors.toList()); + + // Create acting user + AuthenticatedUser actingUser = new AuthenticatedUser(importContext.getUpload().getUpdatedByUser().getName(), new ArrayList<>(), importContext.getUpload().getUpdatedByUser().getId(), new ArrayList<>()); + + return programLocationService.create(actingUser, importContext.getProgram().getId(), locationRequests); } + /** + * Fetch objects required by the workflow from the BrAPI service. + * + * @return List of fetched entities + * @throws ApiException if there is an issue with the API call + */ @Override public List brapiRead() throws ApiException { - return null; + // Get the dbIds of the studies belonging to the required exp units + Set locationDbIds = cache.getStudyByNameNoScope().values().stream().map(pio -> pio.getBrAPIObject().getLocationDbId()).collect(Collectors.toSet()); + + // Get the locations belonging to required exp units + return locationService.fetchLocationsByDbId(locationDbIds, importContext.getProgram()); } + /** + * Commit objects changed by the workflow to the BrAPI service. + * + * @param members List of entities to be updated + * @return List of updated entities + * @throws ApiException if there is an issue with the API call + * @throws IllegalArgumentException if method arguments are invalid + */ @Override public List brapiPut(List members) throws ApiException, IllegalArgumentException { - return null; + return new ArrayList<>(); } + /** + * Remove objects created by the workflow from the BrAPI service. + * + * @param members List of entities to be deleted + * @return true if deletion is successful, false otherwise + * @throws ApiException if there is an issue with the API call + */ @Override public boolean brapiDelete(List members) throws ApiException { + // TODO: implement delete for program locations return false; } + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the BrAPI service. + * + * @param status State of the objects + * @return List of deep copies of entities + * @throws ApiException if there is an issue with the API call + */ @Override public List getBrAPIState(ImportObjectState status) throws ApiException { - return null; + return new ArrayList<>(); } + /** + * For workflow pending import objects of a given state, construct deep copies of the objects from the workflow context. + * + * @param status State of the objects + * @return List of deep copies of entities from workflow context + */ @Override public List copyWorkflowMembers(ImportObjectState status) { - return null; + return experimentUtilities.copyWorkflowCachePendingBrAPIObjects(cache.getLocationByName(), ProgramLocation.class, status); } + /** + * For objects in the workflow context, update any foreign-key fields with values generated by the BrAPI service. + * + * @param members List of entities to be updated + */ @Override public void updateWorkflow(List members) { + // Check if the input list is of type List + if (!experimentUtilities.isPopulated(members, ProgramLocation.class)) { + return; + } + + for (U member : members) { + ProgramLocation location = (ProgramLocation) member; + + // Set the system-generated dbId for each newly created location + cache.getLocationByName().get(location.getName()).getBrAPIObject().setLocationDbId(location.getLocationDbId()); + + // Set the location dbid for cached studies + cache.getStudyByNameNoScope().values().stream() + .filter(study -> location.getId().toString().equals(study.getBrAPIObject().getLocationDbId())) + .forEach(study -> study.getBrAPIObject().setLocationDbId(location.getLocationDbId())); + } } + /** + * Populate the workflow context with objects needed by the workflow. + * + * @param members List of entities to be initialized + */ @Override public void initializeWorkflow(List members) { + // Check if the input list is of type List + if (!experimentUtilities.isPopulated(members, ProgramLocation.class)) { + return; + } + + // Construct the pending locations from the BrAPI locations + List> pendingLocations = members.stream().map((U brapiLocation) -> locationService.constructPIOFromBrapiLocation((ProgramLocation) brapiLocation)).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending location by location name + Map> pendingLocationByName = pendingLocations.stream().collect(Collectors.toMap(pio -> pio.getBrAPIObject().getName(), pio -> pio)); + // Add the map to the context for use in processing import + cache.setLocationByName(pendingLocationByName); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java index cd5e3c6bc..463c7fcdc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java @@ -139,7 +139,7 @@ public void updateWorkflow(List members) { cache.getTrialByNameNoScope().get(createdTrialNameNoScope).getBrAPIObject().setTrialDbId(trial.getTrialDbId()); } - // update trial DbIds in studies for all distinct trials + // Update trial DbIds in studies for all distinct trials cache.getTrialByNameNoScope().values().stream() .filter(Objects::nonNull) .distinct() diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java index 8a6bbcb4f..069cd8f09 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java @@ -4,7 +4,18 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +/** + * ExpUnitMiddleware class extends Middleware class to handle compensating transactions in the context of ExpUnitMiddlewareContext. + */ public abstract class ExpUnitMiddleware extends Middleware { + + /** + * Compensates for an error that occurred in the current local transaction, tagging the error and undoing the previous local transaction. + * + * @param context The context in which the compensation is to be performed. + * @param error The error that occurred and needs to be compensated. + * @return True if the prior local transaction was successfully compensated, false otherwise. + */ @Override public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { // tag an error if it occurred in this local transaction diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreationMiddleware.java similarity index 75% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreationMiddleware.java index 5b266be09..d27906afb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreationMiddleware.java @@ -2,20 +2,11 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPIListSummary; -import org.brapi.v2.model.core.request.BrAPIListNewRequest; -import org.brapi.v2.model.core.response.BrAPIListDetails; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; -import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; -import org.breedinginsight.brapps.importer.model.imports.PendingImport; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.services.exceptions.DoesNotExistException; @@ -29,15 +20,15 @@ @Slf4j @Prototype -public class BrAPILocationCreation extends ExpUnitMiddleware { +public class BrAPILocationCreationMiddleware extends ExpUnitMiddleware { ExperimentUtilities experimentUtilities; - ProgramLocationService locationService; + ProgramLocationService programLocationService; private List newLocations; @Inject - public BrAPILocationCreation(ExperimentUtilities experimentUtilities, ProgramLocationService locationService) { + public BrAPILocationCreationMiddleware(ExperimentUtilities experimentUtilities, ProgramLocationService programLocationService) { this.experimentUtilities = experimentUtilities; - this.locationService = locationService; + this.programLocationService = programLocationService; } @Override public boolean process(ExpUnitMiddlewareContext context) { @@ -55,7 +46,7 @@ public boolean process(ExpUnitMiddlewareContext context) { // Create new locations in brapi service List createdLocations = null; try { - createdLocations = new ArrayList<>(locationService.create(actingUser, context.getImportContext().getProgram().getId(), newLocations)); + createdLocations = new ArrayList<>(programLocationService.create(actingUser, context.getImportContext().getProgram().getId(), newLocations)); // Update the context cache for (ProgramLocation createdLocation : createdLocations) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java new file mode 100644 index 000000000..52691ed93 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -0,0 +1,41 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.LocationCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; + +import java.util.Optional; + +public class LocationCommit extends ExpUnitMiddleware { + private LocationCreation locationCreation; + private Optional createdLocations; + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + try { + locationCreation = new LocationCreation(context); + createdLocations = locationCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + return processNext(context); + } + + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // Tag an error if it occurred in this local transaction + error.tag(this.getClass().getName()); + + // Delete any created locations + createdLocations.ifPresent(BrAPICreation.BrAPICreationState::undo); + + // Undo the prior local transaction + return compensatePrior(context, error); + } + +} From b31e60e34c46162574464acddb4d877c0044a3cd Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 4 Jun 2024 14:35:31 -0400 Subject: [PATCH 088/203] create pending study --- .../appendoverwrite/entity/PendingStudy.java | 168 ++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java new file mode 100644 index 000000000..66f22344a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java @@ -0,0 +1,168 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; + +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class PendingStudy implements ExperimentImportEntity{ + ExpUnitContext cache; + ImportContext importContext; + @Inject + StudyService studyService; + @Inject + BrAPIStudyDAO brAPIStudyDAO; + @Inject + ExperimentUtilities experimentUtilities; + + public PendingStudy(ExpUnitMiddlewareContext context) { + this.cache = context.getExpUnitContext(); + this.importContext = context.getImportContext(); + } + /** + * Create new objects generated by the workflow in the BrAPI service. + * + * @param members List of entities to be created + * @return List of created entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiPost(List members) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + return brAPIStudyDAO.createBrAPIStudies(members, importContext.getProgram().getId(), importContext.getUpload()); + } + + /** + * Fetch objects required by the workflow from the BrAPI service. + * + * @return List of fetched entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiRead() throws ApiException { + // Get the dbIds of the studies belonging to the required exp units + Set studyDbIds = cache.getObservationUnitByNameNoScope().values().stream().map(studyService::getStudyDbIdBelongingToPendingUnit).collect(Collectors.toSet()); + return studyService.fetchBrapiStudiesByDbId(studyDbIds, importContext.getProgram()); + } + + /** + * Commit objects changed by the workflow to the BrAPI service. + * + * @param members List of entities to be updated + * @return List of updated entities + * @throws ApiException if there is an issue with the API call + * @throws IllegalArgumentException if method arguments are invalid + */ + @Override + public List brapiPut(List members) throws ApiException, IllegalArgumentException { + return new ArrayList<>(); + } + + /** + * Remove objects created by the workflow from the BrAPI service. + * + * @param members List of entities to be deleted + * @return true if deletion is successful, false otherwise + * @throws ApiException if there is an issue with the API call + */ + @Override + public boolean brapiDelete(List members) throws ApiException { + // TODO: implement delete study endpoint on BrAPIJavaTestServer + return false; + } + + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the BrAPI service. + * + * @param status State of the objects + * @return List of deep copies of entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List getBrAPIState(ImportObjectState status) throws ApiException { + return new ArrayList<>(); + } + + /** + * For workflow pending import objects of a given state, construct deep copies of the objects from the workflow context. + * + * @param status State of the objects + * @return List of deep copies of entities from workflow context + */ + @Override + public List copyWorkflowMembers(ImportObjectState status) { + return experimentUtilities.copyWorkflowCachePendingBrAPIObjects(cache.getStudyByNameNoScope(), BrAPIStudy.class, status); + } + + /** + * For objects in the workflow context, update any foreign-key fields with values generated by the BrAPI service. + * + * @param members List of entities to be updated + */ + @Override + public void updateWorkflow(List members) { + // Check if the input list is of type List + if (!experimentUtilities.isPopulated(members, BrAPIStudy.class)) { + return; + } + + for (U member : members) { + BrAPIStudy study = (BrAPIStudy) member; + + // set the DbId for each newly created study + String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), importContext.getProgram().getKey()); + cache.getStudyByNameNoScope().get(createdStudy_name_no_key).getBrAPIObject().setStudyDbId(study.getStudyDbId()); + + // Set the study dbId for observation units + cache.getObservationUnitByNameNoScope().values() + .stream() + .filter(obsUnit -> obsUnit.getBrAPIObject() + .getStudyName() + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), importContext.getProgram().getKey()))) + .forEach(obsUnit -> { + obsUnit.getBrAPIObject().setStudyDbId(study.getStudyDbId()); + obsUnit.getBrAPIObject().setTrialDbId(study.getTrialDbId()); + }); + } + } + + /** + * Populate the workflow context with objects needed by the workflow. + * + * @param members List of entities to be initialized + */ + @Override + public void initializeWorkflow(List members) { + // Check if the input list is of type List + if (!experimentUtilities.isPopulated(members, BrAPIStudy.class)) { + return; + } + + // Construct the pending studies from the BrAPI trials + List> pendingStudies = members.stream() + .map(s->(BrAPIStudy) s) + .map(pio -> studyService.constructPIOFromBrapiStudy(pio, importContext.getProgram())).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending study by study name with the program key removed + Map> pendingStudyByNameNoScope = pendingStudies.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKeyAndUnknownAdditionalData(pio.getBrAPIObject().getStudyName(), importContext.getProgram().getKey()), pio -> pio)); + + // Add the map to the context for use in processing import + cache.setStudyByNameNoScope(pendingStudyByNameNoScope); + } +} From 4fac6ea56c00f08a73fc68542c27fa6eb5a70b65 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 4 Jun 2024 14:56:48 -0400 Subject: [PATCH 089/203] create study commit middleware --- .../action/create/BrAPIStudyCreation.java | 26 ++++++++++ .../BrAPIStudyReadWorkflowInitialization.java | 24 ++++++++++ .../middleware/commit/BrAPIStudyCommit.java | 47 +++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIStudyCreation.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIStudyCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIStudyCreation.java new file mode 100644 index 000000000..06eaae555 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIStudyCreation.java @@ -0,0 +1,26 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingStudy; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +@Slf4j +public class BrAPIStudyCreation extends BrAPICreation { + + public BrAPIStudyCreation(ExpUnitMiddlewareContext context) { + super(context); + } + + /** + * Abstract method to get the ExperimentImportEntity based on the ExpUnitMiddlewareContext. + * + * @param context the ExpUnitMiddlewareContext object + * @return the ExperimentImportEntity object + */ + @Override + public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + return new PendingStudy(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java new file mode 100644 index 000000000..2738fa0cb --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java @@ -0,0 +1,24 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; + +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingStudy; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +public class BrAPIStudyReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { + + protected BrAPIStudyReadWorkflowInitialization(ExpUnitMiddlewareContext context) { + super(context); + } + + /** + * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. + * + * @param context The ExpUnitMiddlewareContext providing information about the entity. + * @return The ExperimentImportEntity representing the BrAPI entity being acted on. + */ + @Override + public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + return new PendingStudy(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java new file mode 100644 index 000000000..3aa05f428 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -0,0 +1,47 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import io.micronaut.context.annotation.Prototype; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPIStudyCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; + +import java.util.Optional; + +@Slf4j +@Prototype +@NoArgsConstructor +public class BrAPIStudyCommit extends ExpUnitMiddleware { + private BrAPIStudyCreation brAPIStudyCreation; + private Optional createdBrAPIStudies; + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + try { + brAPIStudyCreation = new BrAPIStudyCreation(context); + createdBrAPIStudies = brAPIStudyCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } + + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // Tag an error if it occurred in this local transaction + error.tag(this.getClass().getName()); + + // Delete any created studies from the BrAPI service + createdBrAPIStudies.ifPresent(BrAPICreation.BrAPICreationState::undo); + + // Undo the prior local transaction + return compensatePrior(context, error); + } +} From 92c43861213e219705d9eced823b6a5711caf1b8 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 4 Jun 2024 17:58:04 -0400 Subject: [PATCH 090/203] create pending dataset --- .../action/create/BrAPIDatasetCreation.java | 30 +++ ...rAPIDatasetReadWorkflowInitialization.java | 29 +++ .../entity/PendingDataset.java | 189 ++++++++++++++++++ .../middleware/commit/BrAPIDatasetCommit.java | 40 ++++ 4 files changed, 288 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java new file mode 100644 index 000000000..1e5aa8f03 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java @@ -0,0 +1,30 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingDataset; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +@Slf4j +public class BrAPIDatasetCreation extends BrAPICreation { + /** + * Constructor for BrAPICreation class. + * + * @param context the ExpUnitMiddlewareContext object + */ + protected BrAPIDatasetCreation(ExpUnitMiddlewareContext context) { + super(context); + } + + /** + * Abstract method to get the ExperimentImportEntity based on the ExpUnitMiddlewareContext. + * + * @param context the ExpUnitMiddlewareContext object + * @return the ExperimentImportEntity object + */ + @Override + public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + return new PendingDataset(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java new file mode 100644 index 000000000..2f2378f90 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java @@ -0,0 +1,29 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; + +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingDataset; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +public class BrAPIDatasetReadWorkflowInitialization extends BrAPIReadWorkflowInitialization{ + /** + * Constructs a new BrAPIReadWorkflowInitialization object with the given ExpUnitMiddlewareContext. + * Initializes the entity based on the provided context. + * + * @param context the ExpUnitMiddlewareContext used for initialization. + */ + protected BrAPIDatasetReadWorkflowInitialization(ExpUnitMiddlewareContext context) { + super(context); + } + + /** + * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. + * + * @param context The ExpUnitMiddlewareContext providing information about the entity. + * @return The ExperimentImportEntity representing the BrAPI entity being acted on. + */ + @Override + public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + return new PendingDataset(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java new file mode 100644 index 000000000..9b01a6a9d --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java @@ -0,0 +1,189 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; + +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.request.BrAPIListNewRequest; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class PendingDataset implements ExperimentImportEntity { + ExpUnitContext cache; + ImportContext importContext; + @Inject + BrAPIListDAO brAPIListDAO; + @Inject + DatasetService datasetService; + @Inject + ExperimentUtilities experimentUtilities; + public PendingDataset(ExpUnitMiddlewareContext context) { + this.cache = context.getExpUnitContext(); + this.importContext = context.getImportContext(); + } + /** + * Create new objects generated by the workflow in the BrAPI service. + * + * @param members List of entities to be created + * @return List of created entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiPost(List members) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + // Construct BrAPI list requests + List requests = members.stream().map(details -> { + BrAPIListNewRequest request = new BrAPIListNewRequest(); + request.setListName(details.getListName()); + request.setListType(details.getListType()); + request.setExternalReferences(details.getExternalReferences()); + request.setAdditionalInfo(details.getAdditionalInfo()); + request.data(details.getData()); + return request; + }).collect(Collectors.toList()); + + // The BrAPI service returns summaries with no data details but with system-generated dbIds + List summaries = brAPIListDAO.createBrAPILists(requests, importContext.getProgram().getId(), importContext.getUpload()); + + // Return the dataset data with system-generated dbId + for (BrAPIListSummary summary : summaries) { + for (BrAPIListDetails member : members) { + if (member.getListName().equals(summary.getListName())) { + member.setListDbId(summary.getListDbId()); + } + } + } + + return members; + } + + /** + * Fetch objects required by the workflow from the BrAPI service. + * + * @return List of fetched entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiRead() throws ApiException { + // Get the id of the dataset belonging to the required exp units + String datasetId = cache.getTrialByNameNoScope().values().iterator().next().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + + // Get the dataset belonging to required exp units + return List.of(datasetService.fetchDatasetById(datasetId, importContext.getProgram()).orElseThrow(ApiException::new)); + } + + /** + * Commit objects changed by the workflow to the BrAPI service. + * + * @param members List of entities to be updated + * @return List of updated entities + * @throws ApiException if there is an issue with the API call + * @throws IllegalArgumentException if method arguments are invalid + */ + @Override + public List brapiPut(List members) throws ApiException, IllegalArgumentException { + return new ArrayList<>(); + } + + /** + * Remove objects created by the workflow from the BrAPI service. + * + * @param members List of entities to be deleted + * @return true if deletion is successful, false otherwise + * @throws ApiException if there is an issue with the API call + */ + @Override + public boolean brapiDelete(List members) throws ApiException { + // TODO: implement delete list for BrAPIJavaTestServer + return false; + } + + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the BrAPI service. + * + * @param status State of the objects + * @return List of deep copies of entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List getBrAPIState(ImportObjectState status) throws ApiException { + return new ArrayList<>(); + } + + /** + * For workflow pending import objects of a given state, construct deep copies of the objects from the workflow context. + * + * @param status State of the objects + * @return List of deep copies of entities from workflow context + */ + @Override + public List copyWorkflowMembers(ImportObjectState status) { + return experimentUtilities.copyWorkflowCachePendingBrAPIObjects(cache.getObsVarDatasetByName(), BrAPIListDetails.class, status); + } + + /** + * For objects in the workflow context, update any foreign-key fields with values generated by the BrAPI service. + * + * @param members List of entities to be updated + */ + @Override + public void updateWorkflow(List members) { + // Check if the input list is of type List + if (!experimentUtilities.isPopulated(members, BrAPITrial.class)) { + return; + } + + for (U member : members) { + BrAPIListDetails dataset = (BrAPIListDetails) member; + + // Update the dataset dbId + cache.getObsVarDatasetByName().get(dataset.getListName()).getBrAPIObject().setListDbId(dataset.getListDbId()); + } + + } + + /** + * Populate the workflow context with objects needed by the workflow. + * + * @param members List of entities to be initialized + */ + @Override + public void initializeWorkflow(List members) { + // Check if the input list is of type List + if (!experimentUtilities.isPopulated(members, BrAPIListDetails.class)) { + return; + } + + // Construct the pending dataset from the BrAPI observation variable list + List> pendingDatasets = members.stream() + .map(m -> (BrAPIListDetails) m) + .map(dataset -> datasetService.constructPIOFromDataset(dataset, importContext.getProgram())) + .collect(Collectors.toList()); + + // Construct a hashmap to look up the pending dataset by dataset name + Map> pendingDatasetByName = pendingDatasets.stream() + .collect(Collectors.toMap(pio -> pio.getBrAPIObject().getListName(),pio -> pio)); + + // Add the map to the context for use in processing import + cache.setObsVarDatasetByName(pendingDatasetByName); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java new file mode 100644 index 000000000..dd70c09d4 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -0,0 +1,40 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPIDatasetCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; + +import java.util.Optional; + +public class BrAPIDatasetCommit extends ExpUnitMiddleware { + private BrAPIDatasetCreation datasetCreation; + private Optional createdDatasets; + @Override + public boolean process(ExpUnitMiddlewareContext context) { + + try { + datasetCreation = new BrAPIDatasetCreation(context); + createdDatasets = datasetCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + return processNext(context); + } + + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // Tag an error if it occurred in this local transaction + error.tag(this.getClass().getName()); + + // Delete any created datasets + createdDatasets.ifPresent(BrAPICreation.BrAPICreationState::undo); + + // Undo the prior local transaction + return compensatePrior(context, error); + } +} From 48b05e73966a6906729b97ae10dd431b218b94d4 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 5 Jun 2024 10:24:47 -0400 Subject: [PATCH 091/203] create pending observation unit --- .../action/create/BrAPIDatasetCreation.java | 2 +- .../create/BrAPIObservationUnitCreation.java | 28 +++ ...rvationUnitReadWorkflowInitialization.java | 29 +++ .../entity/PendingObservationUnit.java | 184 ++++++++++++++++++ .../appendoverwrite/entity/PendingStudy.java | 3 +- .../commit/BrAPIObservationUnitCommit.java | 45 +++++ 6 files changed, 289 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationUnitCreation.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java index 1e5aa8f03..479e8b4e9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java @@ -13,7 +13,7 @@ public class BrAPIDatasetCreation extends BrAPICreation { * * @param context the ExpUnitMiddlewareContext object */ - protected BrAPIDatasetCreation(ExpUnitMiddlewareContext context) { + public BrAPIDatasetCreation(ExpUnitMiddlewareContext context) { super(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationUnitCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationUnitCreation.java new file mode 100644 index 000000000..87b5105d4 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationUnitCreation.java @@ -0,0 +1,28 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; + +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingObservationUnit; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +public class BrAPIObservationUnitCreation extends BrAPICreation { + /** + * Constructor for BrAPICreation class. + * + * @param context the ExpUnitMiddlewareContext object + */ + public BrAPIObservationUnitCreation(ExpUnitMiddlewareContext context) { + super(context); + } + + /** + * Abstract method to get the ExperimentImportEntity based on the ExpUnitMiddlewareContext. + * + * @param context the ExpUnitMiddlewareContext object + * @return the ExperimentImportEntity object + */ + @Override + public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + return new PendingObservationUnit(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java new file mode 100644 index 000000000..8b0e01a5a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java @@ -0,0 +1,29 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; + +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingObservationUnit; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +public class BrAPIObservationUnitReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { + /** + * Constructs a new BrAPIReadWorkflowInitialization object with the given ExpUnitMiddlewareContext. + * Initializes the entity based on the provided context. + * + * @param context the ExpUnitMiddlewareContext used for initialization. + */ + protected BrAPIObservationUnitReadWorkflowInitialization(ExpUnitMiddlewareContext context) { + super(context); + } + + /** + * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. + * + * @param context The ExpUnitMiddlewareContext providing information about the entity. + * @return The ExperimentImportEntity representing the BrAPI entity being acted on. + */ + @Override + public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + return new PendingObservationUnit(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java new file mode 100644 index 000000000..0a2d1966f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java @@ -0,0 +1,184 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; + +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +public class PendingObservationUnit implements ExperimentImportEntity { + ExpUnitContext cache; + ImportContext importContext; + @Inject + BrAPIObservationUnitDAO observationUnitDAO; + @Inject + ObservationUnitService observationUnitService; + @Inject + ExperimentUtilities experimentUtilities; + public PendingObservationUnit(ExpUnitMiddlewareContext context) { + this.cache = context.getExpUnitContext(); + this.importContext = context.getImportContext(); + } + /** + * Create new objects generated by the workflow in the BrAPI service. + * + * @param members List of entities to be created + * @return List of created entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiPost(List members) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + return observationUnitDAO.createBrAPIObservationUnits(members, importContext.getProgram().getId(), importContext.getUpload()); + } + + /** + * Fetch objects required by the workflow from the BrAPI service. + * + * @return List of fetched entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiRead() throws ApiException { + // Collect deltabreed-generated exp unit ids listed in the import + Set expUnitIds = cache.getReferenceOUIds(); + + // For each id fetch the observation unit from the brapi data store + return observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), importContext.getProgram()); + } + + /** + * Commit objects changed by the workflow to the BrAPI service. + * + * @param members List of entities to be updated + * @return List of updated entities + * @throws ApiException if there is an issue with the API call + * @throws IllegalArgumentException if method arguments are invalid + */ + @Override + public List brapiPut(List members) throws ApiException, IllegalArgumentException { + return null; + } + + /** + * Remove objects created by the workflow from the BrAPI service. + * + * @param members List of entities to be deleted + * @return true if deletion is successful, false otherwise + * @throws ApiException if there is an issue with the API call + */ + @Override + public boolean brapiDelete(List members) throws ApiException { + return false; + } + + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the BrAPI service. + * + * @param status State of the objects + * @return List of deep copies of entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List getBrAPIState(ImportObjectState status) throws ApiException { + return null; + } + + /** + * For workflow pending import objects of a given state, construct deep copies of the objects from the workflow context. + * + * @param status State of the objects + * @return List of deep copies of entities from workflow context + */ + @Override + public List copyWorkflowMembers(ImportObjectState status) { + return experimentUtilities.copyWorkflowCachePendingBrAPIObjects(cache.getObservationUnitByNameNoScope(), BrAPIObservationUnit.class, status); + } + + /** + * For objects in the workflow context, update any foreign-key fields with values generated by the BrAPI service. + * + * @param members List of entities to be updated + */ + @Override + public void updateWorkflow(List members) { + // Check if the input list is of type List + if (!experimentUtilities.isPopulated(members, BrAPIObservationUnit.class)) { + return; + } + + for (U member : members) { + BrAPIObservationUnit unit = (BrAPIObservationUnit) member; + + // Set the dbId for observation units + String studyNameNoScope = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), importContext.getProgram().getKey()); + String unitNameNoScope = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getObservationUnitName(), importContext.getProgram().getKey()); + String key = studyNameNoScope + unitNameNoScope; + cache.getObservationUnitByNameNoScope().get(key).getBrAPIObject().setObservationUnitDbId(unit.getObservationUnitDbId()); + + // Set the unit dbId for observations connected with the unit, matching on environment and exp unit + cache.getPendingObservationByHash().values() + .stream() + .filter(obs -> obs.getBrAPIObject() + .getAdditionalInfo() != null + && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null + && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) + .getAsString() + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), importContext.getProgram().getKey())) + && Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getBrAPIObject().getObservationUnitName(), importContext.getProgram().getKey()) + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getObservationUnitName(), importContext.getProgram().getKey())) + ) + .forEach(obs -> { + if (StringUtils.isBlank(obs.getBrAPIObject().getObservationUnitDbId())) { + obs.getBrAPIObject().setObservationUnitDbId(unit.getObservationUnitDbId()); + } + obs.getBrAPIObject().setStudyDbId(unit.getStudyDbId()); + obs.getBrAPIObject().setGermplasmDbId(unit.getGermplasmDbId()); + }); + } + } + + /** + * Populate the workflow context with objects needed by the workflow. + * + * @param members List of entities to be initialized + */ + @Override + public void initializeWorkflow(List members) { + // Check if the input list is of type List + if (!experimentUtilities.isPopulated(members, BrAPIObservationUnit.class)) { + return; + } + + // Construct pending import objects from the units + List> pendingUnits = members.stream().map(u -> (BrAPIObservationUnit) u).map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending unit by ID + Map> pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); + + // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed + Map> pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), importContext.getProgram()); + + // add maps to the context for use in processing import + cache.setPendingObsUnitByOUId(pendingUnitById); + cache.setObservationUnitByNameNoScope(pendingUnitByNameNoScope); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java index 66f22344a..4156b13a3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java @@ -160,7 +160,8 @@ public void initializeWorkflow(List members) { .map(pio -> studyService.constructPIOFromBrapiStudy(pio, importContext.getProgram())).collect(Collectors.toList()); // Construct a hashmap to look up the pending study by study name with the program key removed - Map> pendingStudyByNameNoScope = pendingStudies.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKeyAndUnknownAdditionalData(pio.getBrAPIObject().getStudyName(), importContext.getProgram().getKey()), pio -> pio)); + Map> pendingStudyByNameNoScope = pendingStudies.stream() + .collect(Collectors.toMap(pio -> Utilities.removeProgramKeyAndUnknownAdditionalData(pio.getBrAPIObject().getStudyName(), importContext.getProgram().getKey()), pio -> pio)); // Add the map to the context for use in processing import cache.setStudyByNameNoScope(pendingStudyByNameNoScope); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java new file mode 100644 index 000000000..90caadd7d --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -0,0 +1,45 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPIObservationUnitCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; + +import java.util.Optional; + +public class BrAPIObservationUnitCommit extends ExpUnitMiddleware { + private BrAPIObservationUnitCreation brAPIObservationUnitCreation; + private Optional createdBrAPIObservationUnits; + + /** + * Subclasses will implement this local transaction. + * + * @param context + */ + @Override + public boolean process(ExpUnitMiddlewareContext context) { + try{ + brAPIObservationUnitCreation = new BrAPIObservationUnitCreation(context); + createdBrAPIObservationUnits = brAPIObservationUnitCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + return processNext(context); + } + + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // Tag an error if it occurred in this local transaction + error.tag(this.getClass().getName()); + + // Delete any created trials from the BrAPI service + createdBrAPIObservationUnits.ifPresent(BrAPICreation.BrAPICreationState::undo); + + // Undo the prior local transaction + return compensatePrior(context, error); + } +} From c7ccbb3b4c58a485a8da5230c903f19208e59fe6 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 5 Jun 2024 11:15:24 -0400 Subject: [PATCH 092/203] create pending observation --- .../action/update/BrAPIObservationUpdate.java | 20 +++++++ .../entity/PendingObservation.java | 33 +++++++++- .../commit/BrAPIObservationCommit.java | 60 +++++++++++++++++++ 3 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIObservationUpdate.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIObservationUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIObservationUpdate.java new file mode 100644 index 000000000..aa2649320 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIObservationUpdate.java @@ -0,0 +1,20 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +@Slf4j +public class BrAPIObservationUpdate extends BrAPIUpdate { + + public BrAPIObservationUpdate(ExpUnitMiddlewareContext context) { + super(context); + } + + @Override + public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + return new PendingObservation(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java index 3854a9d9a..b8143f0f6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java @@ -1,14 +1,19 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; +import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.DoesNotExistException; import javax.inject.Inject; import java.util.ArrayList; @@ -24,6 +29,8 @@ public class PendingObservation implements ExperimentImportEntity brapiPost(List members) throws ApiException { - return brAPIObservationDAO.createBrAPIObservations(members, importContext.getProgram().getId(), importContext.getUpload()); + public List brapiPost(List members) throws ApiException, DoesNotExistException { + // TODO: move the trait setting out to a higher level + // Fetch the program traits + List traits = ontologyService.getTraitsByProgramId(importContext.getProgram().getId(), true); + CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); + for ( Trait trait: traits) { + traitMap.put(trait.getObservationVariableName(),trait); + } + + // Set the trait dbId on the observation requests + for (BrAPIObservation observation : members) { + String observationVariableName = observation.getObservationVariableName(); + if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { + String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); + observation.setObservationVariableDbId(observationVariableDbId); + } + } + + // TODO: move this logic out to a higher level + // Do not create observations in the BrAPI service if there is no value + List nonBlankMembers = members.stream().filter(obs -> !obs.getValue().isBlank()).collect(Collectors.toList()); + + // Create the observations + return brAPIObservationDAO.createBrAPIObservations(nonBlankMembers, importContext.getProgram().getId(), importContext.getUpload()); } /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java new file mode 100644 index 000000000..8f232c872 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -0,0 +1,60 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import io.micronaut.context.annotation.Prototype; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPIObservationCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIObservationUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPITrialUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; + +import java.util.Optional; + +@Slf4j +@Prototype +@NoArgsConstructor +public class BrAPIObservationCommit extends ExpUnitMiddleware { + private BrAPIObservationCreation brAPIObservationCreation; + private BrAPIObservationUpdate brAPIObservationUpdate; + private Optional createdBrAPIObservations; + private Optional priorBrAPIObservations; + private Optional updatedObservations; + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + try { + brAPIObservationCreation = new BrAPIObservationCreation(context); + createdBrAPIObservations = brAPIObservationCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); + brAPIObservationUpdate = new BrAPIObservationUpdate(context); + priorBrAPIObservations = brAPIObservationUpdate.getBrAPIState().map(s -> (BrAPIUpdate.BrAPIUpdateState) s); + updatedObservations = brAPIObservationUpdate.execute().map(s -> (BrAPIUpdate.BrAPIUpdateState) s); + + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } + + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // Tag an error if it occurred in this local transaction + error.tag(this.getClass().getName()); + + // Delete any created observations from the BrAPI service + createdBrAPIObservations.ifPresent(BrAPICreation.BrAPICreationState::undo); + + // Revert any changes made to observations in the BrAPI service + priorBrAPIObservations.ifPresent(BrAPIUpdate.BrAPIUpdateState::restore); + + // Undo the prior local transaction + return compensatePrior(context, error); + } +} From ba866cf565a95b3c4aeaa26faa82dc4f709259d8 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 5 Jun 2024 12:36:58 -0400 Subject: [PATCH 093/203] create pending germplasm --- ...PIGermplasmReadWorkflowInitialization.java | 29 ++++ .../entity/PendingGermplasm.java | 138 ++++++++++++++++++ .../entity/PendingObservationUnit.java | 19 +++ 3 files changed, 186 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java new file mode 100644 index 000000000..0b3760206 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java @@ -0,0 +1,29 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; + +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingGermplasm; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +public class BrAPIGermplasmReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { + /** + * Constructs a new BrAPIReadWorkflowInitialization object with the given ExpUnitMiddlewareContext. + * Initializes the entity based on the provided context. + * + * @param context the ExpUnitMiddlewareContext used for initialization. + */ + protected BrAPIGermplasmReadWorkflowInitialization(ExpUnitMiddlewareContext context) { + super(context); + } + + /** + * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. + * + * @param context The ExpUnitMiddlewareContext providing information about the entity. + * @return The ExperimentImportEntity representing the BrAPI entity being acted on. + */ + @Override + public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + return new PendingGermplasm(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java new file mode 100644 index 000000000..51913f57f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java @@ -0,0 +1,138 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; + +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.GermplasmService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +public class PendingGermplasm implements ExperimentImportEntity { + ExpUnitContext cache; + ImportContext importContext; + @Inject + GermplasmService germplasmService; + @Inject + ExperimentUtilities experimentUtilities; + + public PendingGermplasm(ExpUnitMiddlewareContext context) { + this.cache = context.getExpUnitContext(); + this.importContext = context.getImportContext(); + } + /** + * Create new objects generated by the workflow in the BrAPI service. + * + * @param members List of entities to be created + * @return List of created entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiPost(List members) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + return null; + } + + /** + * Fetch objects required by the workflow from the BrAPI service. + * + * @return List of fetched entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiRead() throws ApiException { + // Get the dbIds of the germplasm belonging to the required exp units + Set germplasmDbIds = cache.getObservationUnitByNameNoScope().values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + + // Get the dataset belonging to required exp units + return germplasmService.fetchGermplasmByDbId(new HashSet<>(germplasmDbIds), importContext.getProgram()); + } + + /** + * Commit objects changed by the workflow to the BrAPI service. + * + * @param members List of entities to be updated + * @return List of updated entities + * @throws ApiException if there is an issue with the API call + * @throws IllegalArgumentException if method arguments are invalid + */ + @Override + public List brapiPut(List members) throws ApiException, IllegalArgumentException { + return new ArrayList<>(); + } + + /** + * Remove objects created by the workflow from the BrAPI service. + * + * @param members List of entities to be deleted + * @return true if deletion is successful, false otherwise + * @throws ApiException if there is an issue with the API call + */ + @Override + public boolean brapiDelete(List members) throws ApiException { + return false; + } + + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the BrAPI service. + * + * @param status State of the objects + * @return List of deep copies of entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List getBrAPIState(ImportObjectState status) throws ApiException { + return new ArrayList<>(); + } + + /** + * For workflow pending import objects of a given state, construct deep copies of the objects from the workflow context. + * + * @param status State of the objects + * @return List of deep copies of entities from workflow context + */ + @Override + public List copyWorkflowMembers(ImportObjectState status) { + return experimentUtilities.copyWorkflowCachePendingBrAPIObjects(cache.getExistingGermplasmByGID(), BrAPIGermplasm.class, status); + } + + /** + * For objects in the workflow context, update any foreign-key fields with values generated by the BrAPI service. + * + * @param members List of entities to be updated + */ + @Override + public void updateWorkflow(List members) { + + } + + /** + * Populate the workflow context with objects needed by the workflow. + * + * @param members List of entities to be initialized + */ + @Override + public void initializeWorkflow(List members) { + // Check if the input list is of type List + if (!experimentUtilities.isPopulated(members, BrAPIGermplasm.class)) { + return; + } + + // Construct the pending germplasm from the BrAPI locations + List> pendingGermplasm = members.stream().map(g -> (BrAPIGermplasm) g).map(germplasmService::constructPIOFromBrapiGermplasm).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending germplasm by gid + Map> pendingGermplasmByGID = pendingGermplasm.stream().collect(Collectors.toMap(germplasmService::getGIDFromGermplasmPIO, pio -> pio)); + + // Add the map to the context for use in processing import + cache.setExistingGermplasmByGID(pendingGermplasmByGID); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java index 0a2d1966f..0f14652ad 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java @@ -43,6 +43,25 @@ public PendingObservationUnit(ExpUnitMiddlewareContext context) { */ @Override public List brapiPost(List members) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + // TODO: move the germplasm foreign key assignment out one level + // Set the germplasm foreign key for observation unit requests + cache.getExistingGermplasmByGID().values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(germplasm -> { + + // Collect all observation units that are associated with a germplasm GID + members.stream().filter(obsUnit -> germplasm.getAccessionNumber() != null && + germplasm.getAccessionNumber().equals(obsUnit + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.GID).getAsString())) + + // Set the foreign key for each unit + .forEach(obsUnit -> obsUnit.setGermplasmDbId(germplasm.getGermplasmDbId())); + }); + return observationUnitDAO.createBrAPIObservationUnits(members, importContext.getProgram().getId(), importContext.getUpload()); } From 4742f184cb29dc635ba9c3aaf69cdc85ced7aa1b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 5 Jun 2024 12:53:05 -0400 Subject: [PATCH 094/203] create workflow initialization middleware --- ...rAPIDatasetReadWorkflowInitialization.java | 2 +- ...PIGermplasmReadWorkflowInitialization.java | 2 +- ...rvationUnitReadWorkflowInitialization.java | 2 +- .../BrAPIStudyReadWorkflowInitialization.java | 2 +- .../LocationReadWorkflowInitialization.java | 2 +- .../read/brapi/WorkflowInitialization.java | 46 +++++++++++++++++++ 6 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/WorkflowInitialization.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java index 2f2378f90..8c325bb72 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java @@ -12,7 +12,7 @@ public class BrAPIDatasetReadWorkflowInitialization extends BrAPIReadWorkflowIni * * @param context the ExpUnitMiddlewareContext used for initialization. */ - protected BrAPIDatasetReadWorkflowInitialization(ExpUnitMiddlewareContext context) { + public BrAPIDatasetReadWorkflowInitialization(ExpUnitMiddlewareContext context) { super(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java index 0b3760206..2c45f1b68 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java @@ -12,7 +12,7 @@ public class BrAPIGermplasmReadWorkflowInitialization extends BrAPIReadWorkflowI * * @param context the ExpUnitMiddlewareContext used for initialization. */ - protected BrAPIGermplasmReadWorkflowInitialization(ExpUnitMiddlewareContext context) { + public BrAPIGermplasmReadWorkflowInitialization(ExpUnitMiddlewareContext context) { super(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java index 8b0e01a5a..cd8d20dea 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java @@ -12,7 +12,7 @@ public class BrAPIObservationUnitReadWorkflowInitialization extends BrAPIReadWor * * @param context the ExpUnitMiddlewareContext used for initialization. */ - protected BrAPIObservationUnitReadWorkflowInitialization(ExpUnitMiddlewareContext context) { + public BrAPIObservationUnitReadWorkflowInitialization(ExpUnitMiddlewareContext context) { super(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java index 2738fa0cb..1c06cc3b8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java @@ -7,7 +7,7 @@ public class BrAPIStudyReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { - protected BrAPIStudyReadWorkflowInitialization(ExpUnitMiddlewareContext context) { + public BrAPIStudyReadWorkflowInitialization(ExpUnitMiddlewareContext context) { super(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java index ba563692e..b072e65b5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java @@ -14,7 +14,7 @@ public class LocationReadWorkflowInitialization extends BrAPIReadWorkflowInitial * * @param context the ExpUnitMiddlewareContext used for initialization. */ - protected LocationReadWorkflowInitialization(ExpUnitMiddlewareContext context) { + public LocationReadWorkflowInitialization(ExpUnitMiddlewareContext context) { super(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/WorkflowInitialization.java new file mode 100644 index 000000000..759e39cd8 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/WorkflowInitialization.java @@ -0,0 +1,46 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.*; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; + +import javax.inject.Inject; + +@Slf4j +public class WorkflowInitialization extends ExpUnitMiddleware { + BrAPIObservationUnitReadWorkflowInitialization brAPIObservationUnitReadWorkflowInitialization; + BrAPITrialReadWorkflowInitialization brAPITrialReadWorkflowInitialization; + BrAPIStudyReadWorkflowInitialization brAPIStudyReadWorkflowInitialization; + LocationReadWorkflowInitialization locationReadWorkflowInitialization; + BrAPIDatasetReadWorkflowInitialization brAPIDatasetReadWorkflowInitialization; + BrAPIGermplasmReadWorkflowInitialization brAPIGermplasmReadWorkflowInitialization; + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + brAPIObservationUnitReadWorkflowInitialization = new BrAPIObservationUnitReadWorkflowInitialization(context); + brAPITrialReadWorkflowInitialization = new BrAPITrialReadWorkflowInitialization(context); + brAPIStudyReadWorkflowInitialization = new BrAPIStudyReadWorkflowInitialization(context); + locationReadWorkflowInitialization = new LocationReadWorkflowInitialization(context); + brAPIDatasetReadWorkflowInitialization = new BrAPIDatasetReadWorkflowInitialization(context); + brAPIGermplasmReadWorkflowInitialization = new BrAPIGermplasmReadWorkflowInitialization(context); + + log.debug("reading required BrAPI data from BrAPI service"); + try { + brAPIObservationUnitReadWorkflowInitialization.execute(); + brAPITrialReadWorkflowInitialization.execute(); + brAPIStudyReadWorkflowInitialization.execute(); + locationReadWorkflowInitialization.execute(); + brAPIDatasetReadWorkflowInitialization.execute(); + brAPIGermplasmReadWorkflowInitialization.execute(); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} From 04e616029f44dd9dccfc2a7fc701c9dc037c4437 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 5 Jun 2024 21:05:33 -0400 Subject: [PATCH 095/203] connect workflow selector --- .../model/imports/DomainImportService.java | 2 -- .../AppendOverwritePhenotypesWorkflow.java | 24 ++++++++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 5f89de64a..2e37ef4ca 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -18,13 +18,11 @@ package org.breedinginsight.brapps.importer.model.imports; import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import javax.inject.Inject; import javax.inject.Provider; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index e64a2f88a..99597c793 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -6,7 +6,14 @@ import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.GetExistingBrAPIData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.Transaction; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ValidateAllRowsHaveIDs; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import javax.inject.Inject; import javax.inject.Singleton; import java.util.Optional; @@ -14,9 +21,14 @@ @Singleton public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow { private final ExperimentWorkflowNavigator.Workflow workflow; + private final ExpUnitMiddleware middleware; - public AppendOverwritePhenotypesWorkflow(){ + @Inject + public AppendOverwritePhenotypesWorkflow(Transaction transaction, + ValidateAllRowsHaveIDs validateAllRowsHaveIDs, + GetExistingBrAPIData getExistingBrAPIData){ this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE; + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(transaction, validateAllRowsHaveIDs, getExistingBrAPIData); } @Override @@ -44,6 +56,16 @@ public Optional process(ImportServiceContext context) { } // Start processing the import... + ImportContext importContext = ImportContext.builder() + .upload(context.getUpload()) + .importRows(context.getBrAPIImports()) + .data(context.getData()) + .program(context.getProgram()) + .user(context.getUser()) + .commit(context.isCommit()) + .build(); + ExpUnitMiddlewareContext workflowContext = ExpUnitMiddlewareContext.builder().importContext(importContext).build(); + this.middleware.process(workflowContext); return result; } From 36f02b35452d5ab0ed0772b160c32add00655e9e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 5 Jun 2024 21:20:21 -0400 Subject: [PATCH 096/203] change signature of Middleware to return context --- .../middleware/ExpUnitMiddleware.java | 4 ++-- .../middleware/GetExistingBrAPIData.java | 4 ++-- .../appendoverwrite/middleware/Transaction.java | 4 ++-- .../middleware/ValidateAllRowsHaveIDs.java | 9 ++------- .../middleware/commit/BrAPICommit.java | 2 +- .../middleware/commit/BrAPIDatasetCommit.java | 4 ++-- .../middleware/commit/BrAPIDatasetCreation.java | 6 +----- .../commit/BrAPILocationCreationMiddleware.java | 2 +- .../commit/BrAPIObservationCommit.java | 5 ++--- .../commit/BrAPIObservationUnitCommit.java | 4 ++-- .../commit/BrAPIObservationUpdateMiddleware.java | 2 +- .../middleware/commit/BrAPIStudyCommit.java | 4 ++-- .../middleware/commit/BrAPITrialCommit.java | 4 ++-- .../commit/BrAPITrialUpdateMiddleware.java | 2 +- .../middleware/commit/LocationCommit.java | 4 ++-- .../middleware/process/FieldValidation.java | 8 +------- .../process/ImportPreviewStatistics.java | 2 +- .../process/brapi/NewPendingBrAPIObjects.java | 2 +- .../process/brapi/PendingObservation.java | 2 +- .../middleware/read/brapi/RequiredBrAPIData.java | 3 +-- .../middleware/read/brapi/RequiredDatasets.java | 2 +- .../middleware/read/brapi/RequiredGermplasm.java | 6 +----- .../middleware/read/brapi/RequiredLocations.java | 5 +---- .../read/brapi/RequiredObservationUnits.java | 2 +- .../middleware/read/brapi/RequiredStudies.java | 4 +--- .../middleware/read/brapi/RequiredTrials.java | 9 +-------- .../read/brapi/WorkflowInitialization.java | 4 +--- .../experiment/middleware/Middleware.java | 16 ++++++++-------- 28 files changed, 45 insertions(+), 80 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java index 069cd8f09..bd01c2c1b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java @@ -13,11 +13,11 @@ public abstract class ExpUnitMiddleware extends Middleware createdDatasets; @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { datasetCreation = new BrAPIDatasetCreation(context); @@ -27,7 +27,7 @@ public boolean process(ExpUnitMiddlewareContext context) { } @Override - public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { // Tag an error if it occurred in this local transaction error.tag(this.getClass().getName()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCreation.java index eafca0b4b..77b1a9126 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCreation.java @@ -4,16 +4,12 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIListSummary; -import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; -import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; -import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import java.util.ArrayList; @@ -33,7 +29,7 @@ public BrAPIDatasetCreation(ExperimentUtilities experimentUtilities, BrAPIListDA this.brapiListDAO = brapiListDAO; } @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Construct request newDatasetRequests = experimentUtilities.getNewObjects(context.getPendingData().getObsVarDatasetByName(), BrAPIListDetails.class).stream().map(details -> { BrAPIListNewRequest request = new BrAPIListNewRequest(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreationMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreationMiddleware.java index d27906afb..1e7c900d6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreationMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreationMiddleware.java @@ -31,7 +31,7 @@ public BrAPILocationCreationMiddleware(ExperimentUtilities experimentUtilities, this.programLocationService = programLocationService; } @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Construct request newLocations = experimentUtilities.getNewObjects(context.getPendingData().getLocationByName(), ProgramLocation.class) .stream() diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java index 8f232c872..8664291eb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -7,7 +7,6 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPIObservationCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIObservationUpdate; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPITrialUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; @@ -26,7 +25,7 @@ public class BrAPIObservationCommit extends ExpUnitMiddleware { private Optional updatedObservations; @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { brAPIObservationCreation = new BrAPIObservationCreation(context); createdBrAPIObservations = brAPIObservationCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); @@ -44,7 +43,7 @@ public boolean process(ExpUnitMiddlewareContext context) { } @Override - public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { // Tag an error if it occurred in this local transaction error.tag(this.getClass().getName()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index 90caadd7d..b3fff20d0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -19,7 +19,7 @@ public class BrAPIObservationUnitCommit extends ExpUnitMiddleware { * @param context */ @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try{ brAPIObservationUnitCreation = new BrAPIObservationUnitCreation(context); createdBrAPIObservationUnits = brAPIObservationUnitCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); @@ -32,7 +32,7 @@ public boolean process(ExpUnitMiddlewareContext context) { } @Override - public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { // Tag an error if it occurred in this local transaction error.tag(this.getClass().getName()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUpdateMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUpdateMiddleware.java index edb234d2d..3c8461223 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUpdateMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUpdateMiddleware.java @@ -29,7 +29,7 @@ public BrAPIObservationUpdateMiddleware(ExperimentUtilities experimentUtilities, this.brapiObservationDAO = brapiObservationDAO; } @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { mutatedObservationByDbId = experimentUtilities.getMutationsByObjectId(context.getPendingData().getPendingObservationByHash(), BrAPIObservation::getObservationDbId, BrAPIObservation.class); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java index 3aa05f428..1ad885b6a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -20,7 +20,7 @@ public class BrAPIStudyCommit extends ExpUnitMiddleware { private Optional createdBrAPIStudies; @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { brAPIStudyCreation = new BrAPIStudyCreation(context); createdBrAPIStudies = brAPIStudyCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); @@ -34,7 +34,7 @@ public boolean process(ExpUnitMiddlewareContext context) { } @Override - public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { // Tag an error if it occurred in this local transaction error.tag(this.getClass().getName()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index 898bbc7f0..cf3d8a1aa 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -25,7 +25,7 @@ public class BrAPITrialCommit extends ExpUnitMiddleware { private Optional updatedTrials; @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { brAPITrialCreation = new BrAPITrialCreation(context); createdBrAPITrials = brAPITrialCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); @@ -43,7 +43,7 @@ public boolean process(ExpUnitMiddlewareContext context) { } @Override - public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { // Tag an error if it occurred in this local transaction error.tag(this.getClass().getName()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdateMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdateMiddleware.java index cc5152948..d02771211 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdateMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdateMiddleware.java @@ -28,7 +28,7 @@ public BrAPITrialUpdateMiddleware(ExperimentUtilities experimentUtilities, BrAPI this.brapiTrialDAO = brapiTrialDAO; } @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Construct request mutatedTrialsById = experimentUtilities.getMutationsByObjectId(context.getPendingData().getTrialByNameNoScope(), BrAPITrial::getTrialDbId, BrAPITrial.class); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index 52691ed93..d5e770685 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -14,7 +14,7 @@ public class LocationCommit extends ExpUnitMiddleware { private Optional createdLocations; @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { locationCreation = new LocationCreation(context); createdLocations = locationCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); @@ -27,7 +27,7 @@ public boolean process(ExpUnitMiddlewareContext context) { } @Override - public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { // Tag an error if it occurred in this local transaction error.tag(this.getClass().getName()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java index d28a59e2b..ceba67427 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java @@ -1,18 +1,12 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; -import org.breedinginsight.model.Program; import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; @Slf4j public class FieldValidation extends ExpUnitMiddleware { @@ -24,7 +18,7 @@ public FieldValidation(ObservationUnitService observationUnitService) { } @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java index 8e5a10db7..8847a72a8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java @@ -24,7 +24,7 @@ public ImportPreviewStatistics(Provider newPendingBrAPIO } @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { log.debug("generating import preview statistics"); return this.middleware.process(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java index 9b45bb386..9d0eb7231 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java @@ -15,7 +15,7 @@ public NewPendingBrAPIObjects(PendingObservation pendingObservation) { } @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { log.debug("constructing new pending BrAPI objects"); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java index c618ccc0c..41ab8ac40 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java @@ -77,7 +77,7 @@ public PendingObservation(StudyService studyService, } @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { log.debug("verifying traits listed in import"); // Get all the dynamic columns of the import diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java index 0e5ee2a6e..5cfaf2840 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java @@ -5,7 +5,6 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import javax.inject.Inject; -import javax.inject.Provider; @Slf4j public class RequiredBrAPIData extends ExpUnitMiddleware { @@ -29,7 +28,7 @@ public RequiredBrAPIData(RequiredObservationUnits requiredObservationUnits, } @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { log.debug("reading required BrAPI data from BrAPI service"); return this.middleware.process(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java index 36e8be04e..c540f57db 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java @@ -28,7 +28,7 @@ public RequiredDatasets(DatasetService datasetService) { } @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { Program program; String datasetId; BrAPIListDetails dataset = null; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java index 56393ce8d..c0acc0135 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java @@ -3,16 +3,12 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.GermplasmService; import org.breedinginsight.model.Program; @@ -32,7 +28,7 @@ public RequiredGermplasm(GermplasmService germplasmService) { } @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { Program program; Set germplasmDbIds; List brapiGermplasm = null; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java index 49f3ba54c..845d8b396 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java @@ -4,16 +4,13 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; -import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import java.util.List; @@ -32,7 +29,7 @@ public RequiredLocations(LocationService locationService) { } @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { Program program; Set locationDbIds; List brapiLocations; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java index e9451e9b0..24c3752f8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java @@ -26,7 +26,7 @@ public RequiredObservationUnits(ObservationUnitService observationUnitService) { } @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { Program program; Set expUnitIds; List missingIds; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java index d780d9fc6..3df158e85 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java @@ -4,14 +4,12 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -32,7 +30,7 @@ public RequiredStudies(StudyService studyService) { } @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { Program program; Map> pendingUnitByNameNoScope; Set studyDbIds; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java index 0ef945d72..0b141b3bd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java @@ -3,20 +3,13 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.BrAPITrialReadWorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; -import org.breedinginsight.model.Program; -import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; @Slf4j @Prototype @@ -29,7 +22,7 @@ public RequiredTrials(TrialService trialService) { } @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Nothing to do if there are no required units if (context.getPendingData().getObservationUnitByNameNoScope().size() == 0) { return processNext(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/WorkflowInitialization.java index 759e39cd8..3c7ff4152 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/WorkflowInitialization.java @@ -7,8 +7,6 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; -import javax.inject.Inject; - @Slf4j public class WorkflowInitialization extends ExpUnitMiddleware { BrAPIObservationUnitReadWorkflowInitialization brAPIObservationUnitReadWorkflowInitialization; @@ -19,7 +17,7 @@ public class WorkflowInitialization extends ExpUnitMiddleware { BrAPIGermplasmReadWorkflowInitialization brAPIGermplasmReadWorkflowInitialization; @Override - public boolean process(ExpUnitMiddlewareContext context) { + public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { brAPIObservationUnitReadWorkflowInitialization = new BrAPIObservationUnitReadWorkflowInitialization(context); brAPITrialReadWorkflowInitialization = new BrAPITrialReadWorkflowInitialization(context); brAPIStudyReadWorkflowInitialization = new BrAPIStudyReadWorkflowInitialization(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java index c3c78cdd6..f415a73b8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java @@ -23,31 +23,31 @@ public static Middleware link(Middleware first, Middleware... chain) { /** * Subclasses will implement this local transaction. */ - public abstract boolean process(T context); + public abstract T process(T context); /** * Subclasses will implement this method to handle errors and possibly undo the local transaction. */ - public abstract boolean compensate(T context, MiddlewareError error); + public abstract T compensate(T context, MiddlewareError error); /** * Processes the next local transaction or ends traversing if we're at the * last local transaction of the transaction. */ - protected boolean processNext(T context) { + protected T processNext(T context) { if (next == null) { - return true; + return context; } - return next.process(context); + return (T) next.process(context); } /** * Runs the compensating local transaction for the prior local transaction or ends traversing if * we're at the first local transaction of the transaction. */ - protected boolean compensatePrior(T context, MiddlewareError error) { + protected T compensatePrior(T context, MiddlewareError error) { if (prior == null) { - return true; + return context; } - return prior.compensate(context, error); + return (T) prior.compensate(context, error); } private Middleware getLastLink() { From cdf5517ed25a58143d590b26afbbdfcf55e9770e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 5 Jun 2024 21:58:47 -0400 Subject: [PATCH 097/203] compose import preview response --- .../model/response/ImportPreviewResponse.java | 1 + .../workflow/AppendOverwritePhenotypesWorkflow.java | 11 ++++++++++- .../middleware/process/brapi/PendingObservation.java | 5 ++++- .../appendoverwrite/model/ExpUnitContext.java | 4 ++++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/response/ImportPreviewResponse.java b/src/main/java/org/breedinginsight/brapps/importer/model/response/ImportPreviewResponse.java index b46fc6335..23fec70c6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/response/ImportPreviewResponse.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/response/ImportPreviewResponse.java @@ -17,6 +17,7 @@ package org.breedinginsight.brapps.importer.model.response; +import lombok.Builder; import lombok.Getter; import lombok.Setter; import org.breedinginsight.brapps.importer.model.imports.PendingImport; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index 99597c793..56c402b80 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -2,6 +2,7 @@ import lombok.Getter; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; @@ -15,6 +16,7 @@ import javax.inject.Inject; import javax.inject.Singleton; +import java.util.ArrayList; import java.util.Optional; @Getter @@ -65,7 +67,14 @@ public Optional process(ImportServiceContext context) { .commit(context.isCommit()) .build(); ExpUnitMiddlewareContext workflowContext = ExpUnitMiddlewareContext.builder().importContext(importContext).build(); - this.middleware.process(workflowContext); + ExpUnitMiddlewareContext processedContext = this.middleware.process(workflowContext); + ImportPreviewResponse response = new ImportPreviewResponse(); + response.setStatistics(processedContext.getExpUnitContext().getStatistic().constructPreviewMap()); + response.setRows(new ArrayList<>(processedContext.getImportContext().getMappedBrAPIImport().values())); + response.setDynamicColumnNames(processedContext.getImportContext().getUpload().getDynamicColumnNamesList()); + + result.ifPresent(importWorkflowResult -> importWorkflowResult.setImportPreviewResponse(Optional.of(response))); + return result; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java index 41ab8ac40..46063fc0f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java @@ -278,13 +278,14 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Validate processed data processedData.getValidationErrors().ifPresent(errList -> errList.forEach(e->validationErrors.addError(rowNum, e))); - // Update import preview statistics + // Update import preview statistics and set in the context processedData.updateTally(statistic); statistic.addEnvironmentName(studyName); // TODO: change null values to actual data // TODO: change signature to take two args, studyName and unitName statistic.addObservationUnitId(null); statistic.addGid(null); + context.getExpUnitContext().setStatistic(statistic); // Construct a pending observation PendingImportObject pendingProcessedData = processedData.constructPendingObservation(); @@ -302,6 +303,8 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Add the pending observation map to the context for use in processing the import context.getPendingData().setPendingObservationByHash(pendingObservationByHash); + + return context; } catch (DoesNotExistException | ApiException | UnprocessableEntityException e) { this.compensate(context, new MiddlewareError(() -> { throw new RuntimeException(e); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java index 29aef8a6a..ab7541710 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java @@ -10,6 +10,7 @@ import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; import org.breedinginsight.model.ProgramLocation; import tech.tablesaw.columns.Column; @@ -29,6 +30,9 @@ public class ExpUnitContext { private Map> pendingLocationByOUId = new HashMap<>(); private Map> pendingGermplasmByOUId = new HashMap<>(); + // Processing statistics + private AppendStatistic statistic; + // Carry over from PendingData private Map> observationUnitByNameNoScope; private Map> trialByNameNoScope; From 0a26bcfcf8712edb96efd018d78250d960c812d4 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 6 Jun 2024 15:31:51 -0400 Subject: [PATCH 098/203] connect up middleware --- .../AppendOverwritePhenotypesWorkflow.java | 86 ----------------- .../AppendOverwritePhenotypesWorkflow.java | 95 +++++++++++++++---- .../middleware/Transaction.java | 10 +- .../middleware/commit/BrAPICommit.java | 23 ++++- .../RequiredBrAPIData.java | 2 +- .../RequiredDatasets.java | 2 +- .../RequiredGermplasm.java | 2 +- .../RequiredLocations.java | 2 +- .../RequiredObservationUnits.java | 2 +- .../brapi => initialize}/RequiredStudies.java | 2 +- .../brapi => initialize}/RequiredTrials.java | 2 +- .../WorkflowInitialization.java | 2 +- ...servation.java => ImportTableProcess.java} | 7 +- .../process/brapi/NewPendingBrAPIObjects.java | 5 +- .../AppendOverwritePhenotypesWorkflow.java | 42 ++++++++ .../workflow/ExperimentWorkflowFactory.java | 1 - 16 files changed, 158 insertions(+), 127 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{read/brapi => initialize}/RequiredBrAPIData.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{read/brapi => initialize}/RequiredDatasets.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{read/brapi => initialize}/RequiredGermplasm.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{read/brapi => initialize}/RequiredLocations.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{read/brapi => initialize}/RequiredObservationUnits.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{read/brapi => initialize}/RequiredStudies.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{read/brapi => initialize}/RequiredTrials.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{read/brapi => initialize}/WorkflowInitialization.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/{brapi/PendingObservation.java => ImportTableProcess.java} (98%) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/AppendOverwritePhenotypesWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java deleted file mode 100644 index 56c402b80..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ /dev/null @@ -1,86 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; - -import lombok.Getter; -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.GetExistingBrAPIData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.Transaction; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ValidateAllRowsHaveIDs; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; - -import javax.inject.Inject; -import javax.inject.Singleton; -import java.util.ArrayList; -import java.util.Optional; - -@Getter -@Singleton -public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow { - private final ExperimentWorkflowNavigator.Workflow workflow; - private final ExpUnitMiddleware middleware; - - @Inject - public AppendOverwritePhenotypesWorkflow(Transaction transaction, - ValidateAllRowsHaveIDs validateAllRowsHaveIDs, - GetExistingBrAPIData getExistingBrAPIData){ - this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE; - this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(transaction, validateAllRowsHaveIDs, getExistingBrAPIData); - } - - @Override - public Optional process(ImportServiceContext context) { - // Workflow processing the context - ImportWorkflow workflow = ImportWorkflow.builder() - .urlFragment(getWorkflow().getUrlFragment()) - .displayName(getWorkflow().getDisplayName()) - .build(); - - // No-preview result - Optional result = Optional.of(ImportWorkflowResult.builder() - .workflow(workflow) - .importPreviewResponse(Optional.empty()) - .build()); - - // Skip this workflow unless appending or overwriting observation data - if (context != null && !this.workflow.isEqual(context.getWorkflow())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return result; - } - - // Start processing the import... - ImportContext importContext = ImportContext.builder() - .upload(context.getUpload()) - .importRows(context.getBrAPIImports()) - .data(context.getData()) - .program(context.getProgram()) - .user(context.getUser()) - .commit(context.isCommit()) - .build(); - ExpUnitMiddlewareContext workflowContext = ExpUnitMiddlewareContext.builder().importContext(importContext).build(); - ExpUnitMiddlewareContext processedContext = this.middleware.process(workflowContext); - ImportPreviewResponse response = new ImportPreviewResponse(); - response.setStatistics(processedContext.getExpUnitContext().getStatistic().constructPreviewMap()); - response.setRows(new ArrayList<>(processedContext.getImportContext().getMappedBrAPIImport().values())); - response.setDynamicColumnNames(processedContext.getImportContext().getUpload().getDynamicColumnNamesList()); - - result.ifPresent(importWorkflowResult -> importWorkflowResult.setImportPreviewResponse(Optional.of(response))); - - return result; - } - - @Override - public int getOrder() { - return 2; - } - -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 5db10d5b7..a4089723c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -1,42 +1,99 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite; -import io.micronaut.context.annotation.Prototype; +import lombok.Getter; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.GetExistingBrAPIData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.Transaction; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ValidateAllRowsHaveIDs; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.BrAPICommit; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize.WorkflowInitialization; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.ImportTableProcess; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; -import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.Workflow; import javax.inject.Inject; -import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.ArrayList; +import java.util.Optional; +@Getter +@Singleton +public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow { + private final ExperimentWorkflowNavigator.Workflow workflow; + private final ExpUnitMiddleware middleware; -@Prototype -public class AppendOverwritePhenotypesWorkflow implements Workflow { - - ExpUnitMiddleware middleware; @Inject public AppendOverwritePhenotypesWorkflow(Transaction transaction, ValidateAllRowsHaveIDs validateAllRowsHaveIDs, - GetExistingBrAPIData getExistingBrAPIData) { - - this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(transaction, validateAllRowsHaveIDs, getExistingBrAPIData); + WorkflowInitialization workflowInitialization, + ImportTableProcess importTableProcess, + BrAPICommit brAPICommit){ + this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE; + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( + transaction, + validateAllRowsHaveIDs, + workflowInitialization, + importTableProcess, + brAPICommit); } + @Override - public ProcessedData process(ImportContext context) { - ExpUnitMiddlewareContext workflowContext = ExpUnitMiddlewareContext.builder().importContext(context).build(); - this.middleware.process(workflowContext); + public Optional process(ImportServiceContext context) { + // Workflow processing the context + ImportWorkflow workflow = ImportWorkflow.builder() + .urlFragment(getWorkflow().getUrlFragment()) + .displayName(getWorkflow().getDisplayName()) + .build(); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .workflow(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless appending or overwriting observation data + if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Build the context for processing the import + ImportContext importContext = ImportContext.builder() + .upload(context.getUpload()) + .importRows(context.getBrAPIImports()) + .data(context.getData()) + .program(context.getProgram()) + .user(context.getUser()) + .commit(context.isCommit()) + .build(); + ExpUnitMiddlewareContext workflowContext = ExpUnitMiddlewareContext.builder().importContext(importContext).build(); + + // Process the workflow + ExpUnitMiddlewareContext processedContext = this.middleware.process(workflowContext); + + // Shape and return the workflow response + ImportPreviewResponse response = new ImportPreviewResponse(); + response.setStatistics(processedContext.getExpUnitContext().getStatistic().constructPreviewMap()); + response.setRows(new ArrayList<>(processedContext.getImportContext().getMappedBrAPIImport().values())); + response.setDynamicColumnNames(processedContext.getImportContext().getUpload().getDynamicColumnNamesList()); + + result.ifPresent(importWorkflowResult -> importWorkflowResult.setImportPreviewResponse(Optional.of(response))); - // TODO: implement - return new ProcessedData(); + return result; } @Override - public String getName() { - return "AppendOverwritePhenotypesWorkflow"; + public int getOrder() { + return 2; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java index e8f0a5a15..b4a8dec9c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java @@ -4,10 +4,16 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import javax.inject.Inject; + @Slf4j public class Transaction extends ExpUnitMiddleware { - // TODO: add member for ExpUnitContext + ExpUnitMiddlewareContext middleware; + + @Inject + public Transaction() { + } @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { return processNext(context); @@ -17,6 +23,6 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { // TODO: handle any error here - return true; + return context; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java index ba2bc66f2..b27fb5c20 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java @@ -10,16 +10,29 @@ public class BrAPICommit extends ExpUnitMiddleware { ExpUnitMiddleware middleware; @Inject - public BrAPICommit(ExpUnitMiddleware brAPITrialCreation) { - this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(brAPITrialCreation); + public BrAPICommit(BrAPIDatasetCommit brAPIDatasetCommit, + BrAPITrialCommit brAPITrialCommit, + LocationCommit locationCommit, + BrAPIStudyCommit brAPIStudyCommit, + BrAPIObservationUnitCommit brAPIObservationUnitCommit, + BrAPIObservationCommit brAPIObservationCommit) { + + // TODO: add methods to entity/action classes to register and watch for required foreign key values so order does not have to be hard-wired + // Note: the order is important because system-generated dbIds from prior steps are used as foreign keys in + // subsequent steps + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( + brAPIDatasetCommit, + brAPITrialCommit, + locationCommit, + brAPIStudyCommit, + brAPIObservationUnitCommit, + brAPIObservationCommit); } @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { log.debug("starting post of experiment data to BrAPI server"); - - - return processNext(context); + return this.middleware.process(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredBrAPIData.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredBrAPIData.java index 5cfaf2840..4009a8454 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredBrAPIData.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredDatasets.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredDatasets.java index c540f57db..c15222b9c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredDatasets.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredGermplasm.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredGermplasm.java index c0acc0135..4fab10f19 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredGermplasm.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredLocations.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredLocations.java index 845d8b396..1e8993adc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredLocations.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredObservationUnits.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredObservationUnits.java index 24c3752f8..85cef8184 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredObservationUnits.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredStudies.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredStudies.java index 3df158e85..d77ca0350 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredStudies.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredTrials.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredTrials.java index 0b141b3bd..c179cb37a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredTrials.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/WorkflowInitialization.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index 3c7ff4152..d9d158196 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 46063fc0f..80c74e23f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; import com.google.gson.Gson; import lombok.extern.slf4j.Slf4j; @@ -23,7 +23,6 @@ import org.breedinginsight.brapps.importer.services.FileMappingUtil; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.*; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; @@ -48,7 +47,7 @@ import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.MULTIPLE_EXP_TITLES; @Slf4j -public class PendingObservation extends ExpUnitMiddleware { +public class ImportTableProcess extends ExpUnitMiddleware { StudyService studyService; ObservationVariableService observationVariableService; ObservationService observationService; @@ -59,7 +58,7 @@ public class PendingObservation extends ExpUnitMiddleware { AppendStatistic statistic; @Inject - public PendingObservation(StudyService studyService, + public ImportTableProcess(StudyService studyService, ObservationVariableService observationVariableService, BrAPIObservationDAO brAPIObservationDAO, ObservationService observationService, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java index 9d0eb7231..bf1e181f5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java @@ -2,6 +2,7 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.ImportTableProcess; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import javax.inject.Inject; @@ -10,8 +11,8 @@ public class NewPendingBrAPIObjects extends ExpUnitMiddleware { ExpUnitMiddleware middleware; @Inject - public NewPendingBrAPIObjects(PendingObservation pendingObservation) { - this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(pendingObservation); // Construct new pending observation + public NewPendingBrAPIObjects(ImportTableProcess importTableProcess) { + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(importTableProcess); // Construct new pending observation } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/AppendOverwritePhenotypesWorkflow.java new file mode 100644 index 000000000..544fb0263 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/AppendOverwritePhenotypesWorkflow.java @@ -0,0 +1,42 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.workflow; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.GetExistingBrAPIData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.Transaction; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ValidateAllRowsHaveIDs; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.Workflow; + +import javax.inject.Inject; +import javax.inject.Provider; + + +@Prototype +public class AppendOverwritePhenotypesWorkflow implements Workflow { + + ExpUnitMiddleware middleware; + @Inject + public AppendOverwritePhenotypesWorkflow(Transaction transaction, + ValidateAllRowsHaveIDs validateAllRowsHaveIDs, + GetExistingBrAPIData getExistingBrAPIData) { + + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(transaction, validateAllRowsHaveIDs, getExistingBrAPIData); + } + @Override + public ProcessedData process(ImportContext context) { + ExpUnitMiddlewareContext workflowContext = ExpUnitMiddlewareContext.builder().importContext(context).build(); + this.middleware.process(workflowContext); + + // TODO: implement + return new ProcessedData(); + } + + @Override + public String getName() { + return "AppendOverwritePhenotypesWorkflow"; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java index 0d960b4d3..0d4c7a698 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java @@ -3,7 +3,6 @@ import org.apache.commons.lang3.StringUtils; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.AppendOverwritePhenotypesWorkflow; import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.CreateNewExperimentWorkflow; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; From 2ff1cb66e4d3cf692e125e6a7fb92f99f229046a Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 6 Jun 2024 20:45:08 -0400 Subject: [PATCH 099/203] catch api error in initialization --- .../AppendOverwritePhenotypesWorkflow.java | 6 +++--- .../action/read/BrAPIReadWorkflowInitialization.java | 6 +++--- ...lidateAllRowsHaveIDs.java => ExpUnitIDValidation.java} | 2 +- .../appendoverwrite/middleware/Transaction.java | 5 ----- .../middleware/initialize/WorkflowInitialization.java | 3 ++- .../workflow/AppendOverwritePhenotypesWorkflow.java | 8 +++----- 6 files changed, 12 insertions(+), 18 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{ValidateAllRowsHaveIDs.java => ExpUnitIDValidation.java} (94%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index a4089723c..04a771740 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -9,7 +9,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.Transaction; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ValidateAllRowsHaveIDs; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitIDValidation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.BrAPICommit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize.WorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.ImportTableProcess; @@ -29,14 +29,14 @@ public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow { @Inject public AppendOverwritePhenotypesWorkflow(Transaction transaction, - ValidateAllRowsHaveIDs validateAllRowsHaveIDs, + ExpUnitIDValidation expUnitIDValidation, WorkflowInitialization workflowInitialization, ImportTableProcess importTableProcess, BrAPICommit brAPICommit){ this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE; this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( transaction, - validateAllRowsHaveIDs, + expUnitIDValidation, workflowInitialization, importTableProcess, brAPICommit); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java index f2aa5bfef..f11e1db3e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java @@ -8,6 +8,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.utilities.Utilities; import java.util.List; import java.util.Optional; @@ -45,9 +46,8 @@ public Optional execute() throws ApiException { entity.initializeWorkflow(fetchedMembers); return Optional.of(new BrAPIReadState(fetchedMembers)); } catch(ApiException e) { - // TODO: add specific error messages to entity service - log.error("Error reading..."); - throw new InternalServerException("Error reading...", e); + log.error(String.format("Error fetching %s: %s", entity.getClass().getName(), Utilities.generateApiExceptionLogMessage(e)), e); + throw new ApiException(e); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java similarity index 94% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java index b48643c6b..c57738230 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java @@ -6,7 +6,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; @Slf4j -public class ValidateAllRowsHaveIDs extends ExpUnitMiddleware { +public class ExpUnitIDValidation extends ExpUnitMiddleware { @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java index b4a8dec9c..8526fa707 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java @@ -8,12 +8,7 @@ @Slf4j public class Transaction extends ExpUnitMiddleware { - ExpUnitMiddlewareContext middleware; - @Inject - public Transaction() { - - } @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { return processNext(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index d9d158196..62873005b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; +import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.*; @@ -35,7 +36,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { brAPIGermplasmReadWorkflowInitialization.execute(); } catch (ApiException e) { this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); + throw new InternalServerException(e.toString(), e); })); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/AppendOverwritePhenotypesWorkflow.java index 544fb0263..a682edaca 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/AppendOverwritePhenotypesWorkflow.java @@ -4,14 +4,12 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.GetExistingBrAPIData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.Transaction; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ValidateAllRowsHaveIDs; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitIDValidation; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; -import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.Workflow; import javax.inject.Inject; -import javax.inject.Provider; @Prototype @@ -20,10 +18,10 @@ public class AppendOverwritePhenotypesWorkflow implements Workflow { ExpUnitMiddleware middleware; @Inject public AppendOverwritePhenotypesWorkflow(Transaction transaction, - ValidateAllRowsHaveIDs validateAllRowsHaveIDs, + ExpUnitIDValidation expUnitIDValidation, GetExistingBrAPIData getExistingBrAPIData) { - this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(transaction, validateAllRowsHaveIDs, getExistingBrAPIData); + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(transaction, expUnitIDValidation, getExistingBrAPIData); } @Override public ProcessedData process(ImportContext context) { From 394451a028e92b819d14c4a41cc62f7faf98bda9 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 7 Jun 2024 17:55:32 -0400 Subject: [PATCH 100/203] add middlewareError to expUnitContext --- .../AppendOverwritePhenotypesWorkflow.java | 8 +++++ .../middleware/ExpUnitIDValidation.java | 7 ++-- .../middleware/ExpUnitMiddleware.java | 8 ++--- .../middleware/GetExistingBrAPIData.java | 25 ++++--------- .../middleware/Transaction.java | 6 +--- .../middleware/commit/BrAPIDatasetCommit.java | 11 +++--- .../commit/BrAPIObservationCommit.java | 11 +++--- .../commit/BrAPIObservationUnitCommit.java | 11 +++--- .../middleware/commit/BrAPIStudyCommit.java | 11 +++--- .../middleware/commit/BrAPITrialCommit.java | 11 +++--- .../middleware/commit/LocationCommit.java | 11 +++--- .../initialize/RequiredDatasets.java | 5 +-- .../initialize/RequiredGermplasm.java | 5 +-- .../initialize/RequiredLocations.java | 5 +-- .../initialize/RequiredObservationUnits.java | 5 +-- .../initialize/RequiredStudies.java | 5 +-- .../middleware/initialize/RequiredTrials.java | 5 +-- .../initialize/WorkflowInitialization.java | 6 ++-- .../middleware/process/FieldValidation.java | 5 +-- .../process/ImportTableProcess.java | 35 +++++++++---------- .../appendoverwrite/model/ExpUnitContext.java | 4 +++ .../experiment/middleware/Middleware.java | 8 ++--- .../experiment/model/MiddlewareError.java | 9 +++-- 23 files changed, 90 insertions(+), 127 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 04a771740..e0bcd6e9b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -18,6 +18,7 @@ import javax.inject.Inject; import javax.inject.Singleton; +import java.lang.reflect.Constructor; import java.util.ArrayList; import java.util.Optional; @@ -80,6 +81,13 @@ public Optional process(ImportServiceContext context) { // Process the workflow ExpUnitMiddlewareContext processedContext = this.middleware.process(workflowContext); + // TODO: Rethrow any exceptions caught during processing the context +// Optional.ofNullable(processedContext.getExpUnitContext().getMiddlewareError()).ifPresent(e -> { +// Constructor constructor = e.getError().getClass().getConstructor(String.class, Throwable.class); +// Exception newException = constructor.newInstance(e.getError().getMessage(), e); +// throw newException; +// }); + // Shape and return the workflow response ImportPreviewResponse response = new ImportPreviewResponse(); response.setStatistics(processedContext.getExpUnitContext().getStatistic().constructPreviewMap()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java index c57738230..ea2c48acc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java @@ -3,7 +3,6 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; @Slf4j public class ExpUnitIDValidation extends ExpUnitMiddleware { @@ -15,12 +14,12 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { } @Override - public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { // tag an error if it occurred in this local transaction - error.tag(this.getClass().getName()); + context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); // undo the prior local transaction - return compensatePrior(context, error); + return compensatePrior(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java index bd01c2c1b..4763f10dd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java @@ -2,7 +2,6 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.Middleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; /** * ExpUnitMiddleware class extends Middleware class to handle compensating transactions in the context of ExpUnitMiddlewareContext. @@ -13,15 +12,14 @@ public abstract class ExpUnitMiddleware extends Middleware> fetchReferenceObservationUnits( ExpUnitMiddlewareContext context) { @@ -75,9 +73,7 @@ private Map> fetchReferenceObs () -> { // but throw an error if no unit ID - this.compensate(context, new MiddlewareError(() -> { - throw new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID"); - })); + this.compensate(context); } ); @@ -94,23 +90,14 @@ private Map> fetchReferenceObs missingIds.removeAll(fetchedIds); // throw error reporting any reference IDs with no corresponding stored unit in the brapi data store - this.compensate(context, new MiddlewareError(() -> { - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingIds)); - })); + this.compensate(context); } return pendingUnitById; } catch (ApiException e) { // throw an error if problem getting data from the brapi data store - this.compensate(context, new MiddlewareError(() -> { - log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); - try { - throw new ApiException(e); - } catch (ApiException ex) { - throw new RuntimeException(ex); - } - })); + this.compensate(context); } return pendingUnitById; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java index 8526fa707..e99c7ae40 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java @@ -2,9 +2,6 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; - -import javax.inject.Inject; @Slf4j public class Transaction extends ExpUnitMiddleware { @@ -15,8 +12,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { } @Override - public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { - // TODO: handle any error here + public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { return context; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index 6b0543414..ea0fa2998 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -19,22 +19,21 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { datasetCreation = new BrAPIDatasetCreation(context); createdDatasets = datasetCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); + context.getExpUnitContext().setProcessError(new MiddlewareError(e)); + return this.compensate(context); } return processNext(context); } @Override - public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { // Tag an error if it occurred in this local transaction - error.tag(this.getClass().getName()); + context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); // Delete any created datasets createdDatasets.ifPresent(BrAPICreation.BrAPICreationState::undo); // Undo the prior local transaction - return compensatePrior(context, error); + return compensatePrior(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java index 8664291eb..c76464ba1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -34,18 +34,17 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { updatedObservations = brAPIObservationUpdate.execute().map(s -> (BrAPIUpdate.BrAPIUpdateState) s); } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); + context.getExpUnitContext().setProcessError(new MiddlewareError(e)); + return this.compensate(context); } return processNext(context); } @Override - public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { // Tag an error if it occurred in this local transaction - error.tag(this.getClass().getName()); + context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); // Delete any created observations from the BrAPI service createdBrAPIObservations.ifPresent(BrAPICreation.BrAPICreationState::undo); @@ -54,6 +53,6 @@ public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context, Mid priorBrAPIObservations.ifPresent(BrAPIUpdate.BrAPIUpdateState::restore); // Undo the prior local transaction - return compensatePrior(context, error); + return compensatePrior(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index b3fff20d0..b4b433e7e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -24,22 +24,21 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { brAPIObservationUnitCreation = new BrAPIObservationUnitCreation(context); createdBrAPIObservationUnits = brAPIObservationUnitCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); + context.getExpUnitContext().setProcessError(new MiddlewareError(e)); + return this.compensate(context); } return processNext(context); } @Override - public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { // Tag an error if it occurred in this local transaction - error.tag(this.getClass().getName()); + context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); // Delete any created trials from the BrAPI service createdBrAPIObservationUnits.ifPresent(BrAPICreation.BrAPICreationState::undo); // Undo the prior local transaction - return compensatePrior(context, error); + return compensatePrior(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java index 1ad885b6a..2ff3fa1b5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -25,23 +25,22 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { brAPIStudyCreation = new BrAPIStudyCreation(context); createdBrAPIStudies = brAPIStudyCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); + context.getExpUnitContext().setProcessError(new MiddlewareError(e)); + return this.compensate(context); } return processNext(context); } @Override - public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { // Tag an error if it occurred in this local transaction - error.tag(this.getClass().getName()); + context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); // Delete any created studies from the BrAPI service createdBrAPIStudies.ifPresent(BrAPICreation.BrAPICreationState::undo); // Undo the prior local transaction - return compensatePrior(context, error); + return compensatePrior(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index cf3d8a1aa..3b8140f30 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -34,18 +34,17 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { updatedTrials = brAPITrialUpdate.execute().map(s -> (BrAPIUpdate.BrAPIUpdateState) s); } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); + context.getExpUnitContext().setProcessError(new MiddlewareError(e)); + return this.compensate(context); } return processNext(context); } @Override - public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { // Tag an error if it occurred in this local transaction - error.tag(this.getClass().getName()); + context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); // Delete any created trials from the BrAPI service createdBrAPITrials.ifPresent(BrAPICreation.BrAPICreationState::undo); @@ -54,6 +53,6 @@ public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context, Mid priorBrAPITrials.ifPresent(BrAPIUpdate.BrAPIUpdateState::restore); // Undo the prior local transaction - return compensatePrior(context, error); + return compensatePrior(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index d5e770685..92de93cae 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -19,23 +19,22 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { locationCreation = new LocationCreation(context); createdLocations = locationCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); + context.getExpUnitContext().setProcessError(new MiddlewareError(e)); + return this.compensate(context); } return processNext(context); } @Override - public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { // Tag an error if it occurred in this local transaction - error.tag(this.getClass().getName()); + context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); // Delete any created locations createdLocations.ifPresent(BrAPICreation.BrAPICreationState::undo); // Undo the prior local transaction - return compensatePrior(context, error); + return compensatePrior(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredDatasets.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredDatasets.java index c15222b9c..f9f773fb2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredDatasets.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredDatasets.java @@ -9,7 +9,6 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; import org.breedinginsight.model.Program; @@ -65,9 +64,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Add the map to the context for use in processing import context.getPendingData().setObsVarDatasetByName(pendingDatasetByName); } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); + this.compensate(context); } return processNext(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredGermplasm.java index 4fab10f19..096650dae 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredGermplasm.java @@ -8,7 +8,6 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.GermplasmService; import org.breedinginsight.model.Program; @@ -60,9 +59,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Add the map to the context for use in processing import context.getPendingData().setExistingGermplasmByGID(pendingGermplasmByGID); } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); + this.compensate(context); } return processNext(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredLocations.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredLocations.java index 1e8993adc..e3d8643c4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredLocations.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredLocations.java @@ -7,7 +7,6 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; @@ -61,9 +60,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Add the map to the context for use in processing import context.getPendingData().setLocationByName(pendingLocationByName); } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); + this.compensate(context); } return processNext(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredObservationUnits.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredObservationUnits.java index 85cef8184..99624bd46 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredObservationUnits.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredObservationUnits.java @@ -7,7 +7,6 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; import org.breedinginsight.model.Program; @@ -57,9 +56,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); + this.compensate(context); } return processNext(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredStudies.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredStudies.java index d77ca0350..59a434e4a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredStudies.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredStudies.java @@ -8,7 +8,6 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -63,9 +62,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Add the map to the context for use in processing import context.getPendingData().setStudyByNameNoScope(pendingStudyByNameNoScope); } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); + this.compensate(context); } return processNext(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredTrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredTrials.java index c179cb37a..374271d0c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredTrials.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredTrials.java @@ -6,7 +6,6 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.BrAPITrialReadWorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; import javax.inject.Inject; @@ -34,9 +33,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { brAPITrialReadWorkflowInitialization.execute(); } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); + this.compensate(context); } return processNext(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index 62873005b..ed1a89471 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -1,6 +1,5 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; -import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.*; @@ -35,9 +34,8 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { brAPIDatasetReadWorkflowInitialization.execute(); brAPIGermplasmReadWorkflowInitialization.execute(); } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new InternalServerException(e.toString(), e); - })); + context.getExpUnitContext().setProcessError(new MiddlewareError(e)); + return this.compensate(context); } return processNext(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java index ceba67427..300da17be 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java @@ -3,7 +3,6 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; import javax.inject.Inject; @@ -23,9 +22,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { } catch (Exception e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); + this.compensate(context); } return processNext(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 80c74e23f..0c471ccc1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -34,6 +34,7 @@ import org.breedinginsight.model.Trait; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.services.exceptions.ValidatorException; import org.breedinginsight.utilities.Utilities; import tech.tablesaw.api.Table; import tech.tablesaw.columns.Column; @@ -102,20 +103,18 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { tsValErrs.forEach(validationError -> validationErrors.addError(rowNum, validationError)); } - // Stop processing the import if there are unmatched timestamp columns - if (tsValErrs.size() > 0) { - this.compensate(context, new MiddlewareError(() -> { - // any handling... - })); - } + try { + // Stop processing the import if there are unmatched timestamp columns + if (tsValErrs.size() > 0) { + throw new UnprocessableEntityException("One or more timestamp columns do not have a matching observation variable"); + } - //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval - Map> tsColByPheno = timestampCols.stream().collect(Collectors.toMap(col -> col.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), col -> col)); + //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval + Map> tsColByPheno = timestampCols.stream().collect(Collectors.toMap(col -> col.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), col -> col)); - // Add the map to the context for use in processing import - context.getPendingData().setTimeStampColByPheno(tsColByPheno); + // Add the map to the context for use in processing import + context.getPendingData().setTimeStampColByPheno(tsColByPheno); - try { // Fetch the traits named in the observation variable columns Program program = context.getImportContext().getProgram(); List traits = observationVariableService.fetchTraitsByName(varNames, program); @@ -276,6 +275,9 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Validate processed data processedData.getValidationErrors().ifPresent(errList -> errList.forEach(e->validationErrors.addError(rowNum, e))); + if (validationErrors.hasErrors()) { + throw new ValidatorException(validationErrors); + } // Update import preview statistics and set in the context processedData.updateTally(statistic); @@ -303,13 +305,10 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Add the pending observation map to the context for use in processing the import context.getPendingData().setPendingObservationByHash(pendingObservationByHash); - return context; - } catch (DoesNotExistException | ApiException | UnprocessableEntityException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); + return processNext(context); + } catch (DoesNotExistException | ApiException | UnprocessableEntityException | ValidatorException e) { + context.getExpUnitContext().setProcessError(new MiddlewareError(e)); + return this.compensate(context); } - - return processNext(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java index ab7541710..000ad0680 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java @@ -11,6 +11,7 @@ import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.model.ProgramLocation; import tech.tablesaw.columns.Column; @@ -33,6 +34,9 @@ public class ExpUnitContext { // Processing statistics private AppendStatistic statistic; + // Exceptions + private MiddlewareError processError; + // Carry over from PendingData private Map> observationUnitByNameNoScope; private Map> trialByNameNoScope; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java index f415a73b8..b4757cb17 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java @@ -1,7 +1,5 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.middleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; - public abstract class Middleware { Middleware next; @@ -27,7 +25,7 @@ public static Middleware link(Middleware first, Middleware... chain) { /** * Subclasses will implement this method to handle errors and possibly undo the local transaction. */ - public abstract T compensate(T context, MiddlewareError error); + public abstract T compensate(T context); /** * Processes the next local transaction or ends traversing if we're at the * last local transaction of the transaction. @@ -43,11 +41,11 @@ protected T processNext(T context) { * Runs the compensating local transaction for the prior local transaction or ends traversing if * we're at the first local transaction of the transaction. */ - protected T compensatePrior(T context, MiddlewareError error) { + protected T compensatePrior(T context) { if (prior == null) { return context; } - return (T) prior.compensate(context, error); + return (T) prior.compensate(context); } private Middleware getLastLink() { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java index 49960ae3a..d74596f28 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java @@ -7,10 +7,13 @@ public class MiddlewareError { @Getter @Setter String localTransactionName; - Runnable handler; - public MiddlewareError(Runnable handler) { - this.handler = handler; + @Getter + @Setter + Exception error; + + public MiddlewareError(Exception error) { + this.error = error; } public void tag(String name) { From cee306bba6a967e7d676f91eca0d8c55d980aeb4 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Sat, 8 Jun 2024 12:19:29 -0400 Subject: [PATCH 101/203] fix dependency injection circular reference --- .../experiment/ExperimentUtilities.java | 33 ++++++++++++++----- .../entity/PendingDataset.java | 8 ++--- .../entity/PendingGermplasm.java | 4 +-- .../entity/PendingLocation.java | 7 ++-- .../entity/PendingObservation.java | 7 ++-- .../entity/PendingObservationUnit.java | 4 +-- .../appendoverwrite/entity/PendingStudy.java | 6 ++-- .../appendoverwrite/entity/PendingTrial.java | 6 ++-- .../middleware/ExpUnitIDValidation.java | 2 ++ .../middleware/Transaction.java | 2 ++ .../middleware/commit/BrAPICommit.java | 2 ++ .../middleware/commit/BrAPIDatasetCommit.java | 2 ++ .../commit/BrAPIObservationUnitCommit.java | 2 ++ .../middleware/commit/LocationCommit.java | 2 ++ .../initialize/WorkflowInitialization.java | 2 ++ .../process/ImportTableProcess.java | 13 ++++---- .../service/ObservationService.java | 2 ++ 17 files changed, 66 insertions(+), 38 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 17a366872..dcee2f9e3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -6,6 +6,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.exceptions.HttpStatusException; import io.reactivex.functions.Function; +import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.brapi.v2.model.core.BrAPIStudy; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; @@ -17,7 +18,6 @@ import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import javax.inject.Inject; import javax.inject.Singleton; import java.util.*; import java.util.stream.Collectors; @@ -28,18 +28,14 @@ public class ExperimentUtilities { public static final CharSequence COMMA_DELIMITER = ","; public static final String TIMESTAMP_PREFIX = "TS:"; - @Inject Gson gson; - public ExperimentUtilities(Gson gson) { - this.gson = gson; + public ExperimentUtilities() { + this.gson = new Gson(); } - public boolean isPopulated(List list, Class clazz) { + public boolean isInvalidMemberListForClass(List list, Class clazz) { // Check if the input list is of type - if (list == null || list.isEmpty() || !clazz.isInstance(list.get(0))) { - return false; - } - return true; + return list == null || list.isEmpty() || !list.stream().allMatch(clazz::isInstance); } public Optional clone(T obj, Class clazz) { @@ -173,4 +169,23 @@ public static Set collateReferenceOUIds(ExpUnitMiddlewareContext context } return referenceOUIds; } + + public List sortByField(List sortedFields, List unsortedItems, Function fieldGetter) { + CaseInsensitiveMap sortOrder = new CaseInsensitiveMap<>(); + for (int i = 0; i < sortedFields.size(); i++) { + sortOrder.put(sortedFields.get(i), i); + } + + unsortedItems.sort((i1, i2) -> { + try { + String field1 = fieldGetter.apply(i1); + String field2 = fieldGetter.apply(i2); + return Integer.compare(sortOrder.get(field1), sortOrder.get(field2)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + return unsortedItems; + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java index 9b01a6a9d..b9b9b7abf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java @@ -2,7 +2,6 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIListSummary; -import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; @@ -20,7 +19,6 @@ import javax.inject.Inject; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -147,8 +145,8 @@ public List copyWorkflowMembers(ImportObjectState status) { */ @Override public void updateWorkflow(List members) { - // Check if the input list is of type List - if (!experimentUtilities.isPopulated(members, BrAPITrial.class)) { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIListDetails.class)) { return; } @@ -169,7 +167,7 @@ public void updateWorkflow(List members) { @Override public void initializeWorkflow(List members) { // Check if the input list is of type List - if (!experimentUtilities.isPopulated(members, BrAPIListDetails.class)) { + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIListDetails.class)) { return; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java index 51913f57f..28e28d52e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java @@ -121,8 +121,8 @@ public void updateWorkflow(List members) { */ @Override public void initializeWorkflow(List members) { - // Check if the input list is of type List - if (!experimentUtilities.isPopulated(members, BrAPIGermplasm.class)) { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIGermplasm.class)) { return; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java index 8e2f02269..d934c9ec6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java @@ -1,7 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPITrial; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -131,8 +130,8 @@ public List copyWorkflowMembers(ImportObjectState status) { */ @Override public void updateWorkflow(List members) { - // Check if the input list is of type List - if (!experimentUtilities.isPopulated(members, ProgramLocation.class)) { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, ProgramLocation.class)) { return; } @@ -158,7 +157,7 @@ public void updateWorkflow(List members) { @Override public void initializeWorkflow(List members) { // Check if the input list is of type List - if (!experimentUtilities.isPopulated(members, ProgramLocation.class)) { + if (experimentUtilities.isInvalidMemberListForClass(members, ProgramLocation.class)) { return; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java index b8143f0f6..bd0dae932 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java @@ -5,7 +5,6 @@ import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; @@ -95,7 +94,7 @@ public List brapiRead() throws ApiException { @Override public List brapiPut(List members) throws ApiException, IllegalArgumentException { // Check if the input list is of type List - if (!experimentUtilities.isPopulated(members, BrAPIObservation.class)) { + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIObservation.class)) { return new ArrayList(); } @@ -153,7 +152,7 @@ public List copyWorkflowMembers(ImportObjectState status) { @Override public void updateWorkflow(List members) { // Check if the input list is of type List - if (!experimentUtilities.isPopulated(members, BrAPIObservation.class)) { + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIObservation.class)) { return; } @@ -172,7 +171,7 @@ public void updateWorkflow(List members) { @Override public void initializeWorkflow(List members) { // Check if the input list is of type List - if (!experimentUtilities.isPopulated(members, BrAPIObservation.class)) { + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIObservation.class)) { return; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java index 0f14652ad..a10681e7c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java @@ -136,7 +136,7 @@ public List copyWorkflowMembers(ImportObjectState status) @Override public void updateWorkflow(List members) { // Check if the input list is of type List - if (!experimentUtilities.isPopulated(members, BrAPIObservationUnit.class)) { + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIObservationUnit.class)) { return; } @@ -183,7 +183,7 @@ public void updateWorkflow(List members) { @Override public void initializeWorkflow(List members) { // Check if the input list is of type List - if (!experimentUtilities.isPopulated(members, BrAPIObservationUnit.class)) { + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIObservationUnit.class)) { return; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java index 4156b13a3..faaf18615 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java @@ -117,8 +117,8 @@ public List copyWorkflowMembers(ImportObjectState status) { */ @Override public void updateWorkflow(List members) { - // Check if the input list is of type List - if (!experimentUtilities.isPopulated(members, BrAPIStudy.class)) { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIStudy.class)) { return; } @@ -150,7 +150,7 @@ public void updateWorkflow(List members) { @Override public void initializeWorkflow(List members) { // Check if the input list is of type List - if (!experimentUtilities.isPopulated(members, BrAPIStudy.class)) { + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIStudy.class)) { return; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java index 463c7fcdc..6a9a5e809 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java @@ -70,7 +70,7 @@ public List brapiRead() throws ApiException { @Override public List brapiPut(List members) throws ApiException, IllegalArgumentException { // Check if the input list is of type List - if (!experimentUtilities.isPopulated(members, BrAPITrial.class)) { + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPITrial.class)) { return new ArrayList(); } @@ -128,7 +128,7 @@ public List copyWorkflowMembers(ImportObjectState status) { @Override public void updateWorkflow(List members) { // Check if the input list is of type List - if (!experimentUtilities.isPopulated(members, BrAPITrial.class)) { + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPITrial.class)) { return; } @@ -160,7 +160,7 @@ public void updateWorkflow(List members) { @Override public void initializeWorkflow(List members) { // Check if the input list is of type List - if (!experimentUtilities.isPopulated(members, BrAPITrial.class)) { + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPITrial.class)) { return; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java index ea2c48acc..5848b1dbd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java @@ -1,10 +1,12 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; +import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; @Slf4j +@Prototype public class ExpUnitIDValidation extends ExpUnitMiddleware { @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java index e99c7ae40..50006293e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java @@ -1,9 +1,11 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; +import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; @Slf4j +@Prototype public class Transaction extends ExpUnitMiddleware { @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java index b27fb5c20..076e18c54 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; +import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; @@ -7,6 +8,7 @@ import javax.inject.Inject; @Slf4j +@Prototype public class BrAPICommit extends ExpUnitMiddleware { ExpUnitMiddleware middleware; @Inject diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index ea0fa2998..388d701d6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; +import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPIDatasetCreation; @@ -9,6 +10,7 @@ import java.util.Optional; +@Prototype public class BrAPIDatasetCommit extends ExpUnitMiddleware { private BrAPIDatasetCreation datasetCreation; private Optional createdDatasets; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index b4b433e7e..7e54e9b14 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; +import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPIObservationUnitCreation; @@ -9,6 +10,7 @@ import java.util.Optional; +@Prototype public class BrAPIObservationUnitCommit extends ExpUnitMiddleware { private BrAPIObservationUnitCreation brAPIObservationUnitCreation; private Optional createdBrAPIObservationUnits; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index 92de93cae..ab5cbf8b4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; +import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.LocationCreation; @@ -9,6 +10,7 @@ import java.util.Optional; +@Prototype public class LocationCommit extends ExpUnitMiddleware { private LocationCreation locationCreation; private Optional createdLocations; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index ed1a89471..cf921f103 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; +import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.*; @@ -8,6 +9,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; @Slf4j +@Prototype public class WorkflowInitialization extends ExpUnitMiddleware { BrAPIObservationUnitReadWorkflowInitialization brAPIObservationUnitReadWorkflowInitialization; BrAPITrialReadWorkflowInitialization brAPITrialReadWorkflowInitialization; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 0c471ccc1..e8413991d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -1,6 +1,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; import com.google.gson.Gson; +import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; @@ -48,12 +49,13 @@ import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.MULTIPLE_EXP_TITLES; @Slf4j +@Prototype public class ImportTableProcess extends ExpUnitMiddleware { StudyService studyService; ObservationVariableService observationVariableService; ObservationService observationService; BrAPIObservationDAO brAPIObservationDAO; - FileMappingUtil fileMappingUtil; + ExperimentUtilities experimentUtil; Gson gson; FieldValidator fieldValidator; AppendStatistic statistic; @@ -63,16 +65,15 @@ public ImportTableProcess(StudyService studyService, ObservationVariableService observationVariableService, BrAPIObservationDAO brAPIObservationDAO, ObservationService observationService, - FileMappingUtil fileMappingUtil, - Gson gson, + ExperimentUtilities experimentUtil, FieldValidator fieldValidator, AppendStatistic statistic) { this.studyService = studyService; this.observationVariableService = observationVariableService; this.brAPIObservationDAO = brAPIObservationDAO; this.observationService = observationService; - this.fileMappingUtil = fileMappingUtil; - this.gson = gson; + this.experimentUtil = experimentUtil; + this.gson = new Gson(); this.statistic = statistic; } @@ -130,7 +131,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { ); // Sort the traits to match the order of the headers in the import file - List sortedTraits = fileMappingUtil.sortByField(List.copyOf(varNames), new ArrayList<>(traits), TraitEntity::getObservationVariableName); + List sortedTraits = experimentUtil.sortByField(List.copyOf(varNames), new ArrayList<>(traits), TraitEntity::getObservationVariableName); // Get the pending observation dataset PendingImportObject pendingTrial = ExperimentUtilities.getSingleEntryValue(context.getPendingData().getTrialByNameNoScope()).orElseThrow(()->new UnprocessableEntityException(MULTIPLE_EXP_TITLES.getValue())); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index 4490cab30..35275eb20 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -26,6 +26,7 @@ import org.breedinginsight.utilities.Utilities; import tech.tablesaw.columns.Column; +import javax.inject.Singleton; import java.math.BigDecimal; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; @@ -33,6 +34,7 @@ import java.util.*; import java.util.stream.Collectors; +@Singleton public class ObservationService { public boolean validCategory(List categories, String value) { Set categoryValues = categories.stream() From 8825462c5c1d460d03c4cd80746b8307cc31215a Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 10 Jun 2024 10:44:22 -0400 Subject: [PATCH 102/203] Renamed common services --- .../importer/services/pipeline/Pipeline.java | 35 ------------------- ...ulateModifiedPendingImportObjectsStep.java | 2 +- .../CommitPendingImportObjectsStep.java} | 11 +++--- ...va => ExperimentPendingImportService.java} | 0 ...e.java => ExperimentPhenotypeService.java} | 0 ...vice.java => ExperimentSeasonService.java} | 0 ...rvice.java => ExperimentStudyService.java} | 0 ...rvice.java => ExperimentTrialService.java} | 0 ...ce.java => ExperimentValidateService.java} | 0 9 files changed, 8 insertions(+), 40 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/pipeline/Pipeline.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{create => append}/workflow/steps/PopulateModifiedPendingImportObjectsStep.java (74%) rename src/main/java/org/breedinginsight/brapps/importer/services/{pipeline/ProcessingStep.java => processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java} (59%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/{SharedPendingImportService.java => ExperimentPendingImportService.java} (100%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/{SharedPhenotypeService.java => ExperimentPhenotypeService.java} (100%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/{SharedSeasonService.java => ExperimentSeasonService.java} (100%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/{SharedStudyService.java => ExperimentStudyService.java} (100%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/{SharedTrialService.java => ExperimentTrialService.java} (100%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/{SharedValidateService.java => ExperimentValidateService.java} (100%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/Pipeline.java b/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/Pipeline.java deleted file mode 100644 index 76c4433ee..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/Pipeline.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.services.pipeline; - -public class Pipeline { - - private final ProcessingStep currentStep; - - public Pipeline(ProcessingStep currentStep) { - this.currentStep = currentStep; - } - - public Pipeline addProcessingStep(ProcessingStep newStep) { - return new Pipeline<>(input -> newStep.process(currentStep.process(input))); - } - - public O execute(I input) { - return currentStep.process(input); - } -} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateModifiedPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java similarity index 74% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateModifiedPendingImportObjectsStep.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java index a04d33540..e05b1cf25 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateModifiedPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java @@ -1,5 +1,5 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; -// TODO: think this would be for other workflow +// TODO: think this would be for other workflow only public class PopulateModifiedPendingImportObjectsStep { } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java similarity index 59% rename from src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index fb7f5e514..02ab60e3e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -14,9 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; -package org.breedinginsight.brapps.importer.services.pipeline; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; -public interface ProcessingStep { - O process(I input); -} \ No newline at end of file +public class CommitPendingImportDataStep implements ProcessingStep { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPendingImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPendingImportService.java similarity index 100% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPendingImportService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPendingImportService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPhenotypeService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPhenotypeService.java similarity index 100% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPhenotypeService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPhenotypeService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedSeasonService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java similarity index 100% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedSeasonService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentStudyService.java similarity index 100% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentStudyService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java similarity index 100% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java similarity index 100% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedValidateService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java From 2b0256df11a8044c654bafd52d798dce3edf7fe6 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 10 Jun 2024 10:45:36 -0400 Subject: [PATCH 103/203] Simplified workflow step calling --- .../model/workflow/ImportContext.java | 1 + .../model/workflow/ProcessedData.java | 6 + .../importer/model/workflow/Workflow.java | 4 +- .../services/ImportStatusService.java | 4 + .../services/processors/ProcessorData.java | 8 +- .../experiment/ExperimentUtilities.java | 13 + ...ulateModifiedPendingImportObjectsStep.java | 2 +- .../experiment/create/model/PendingData.java | 3 + .../workflow/CreateNewExperimentWorkflow.java | 120 +++- .../CreatePendingImportPopulator.java | 19 +- .../steps/CommitPendingImportObjectsStep.java | 58 +- ...ulateExistingPendingImportObjectsStep.java | 101 ++- .../PopulateNewPendingImportObjectsStep.java | 599 +++++++++++++++++- .../create/workflow/steps/ProcessStep.java | 41 +- .../ExperimentPendingImportService.java | 2 +- .../services/ExperimentPhenotypeService.java | 10 +- .../services/ExperimentSeasonService.java | 4 +- .../services/ExperimentStudyService.java | 6 +- .../services/ExperimentTrialService.java | 8 +- .../services/ExperimentValidateService.java | 4 +- .../PendingImportObjectPopulator.java | 8 +- 21 files changed, 927 insertions(+), 94 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java index 121ef6b19..109c300cc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java @@ -39,6 +39,7 @@ public class ImportContext { private UUID workflowId; private ImportUpload upload; private List importRows; + // TODO: move this out potentially private Map mappedBrAPIImport; private Table data; private Program program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java index f9f8196c2..73302bdb9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -18,8 +18,13 @@ package org.breedinginsight.brapps.importer.model.workflow; import lombok.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; +import java.util.ArrayList; +import java.util.List; import java.util.Map; @Data @@ -27,4 +32,5 @@ @NoArgsConstructor public class ProcessedData { private Map statistics; + private Map mappedBrAPIImport; } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java index 9ea0e7bf9..8b2165933 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -17,6 +17,8 @@ package org.breedinginsight.brapps.importer.model.workflow; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; + public interface Workflow { /** @@ -25,7 +27,7 @@ public interface Workflow { * @param context the import context containing the necessary data for processing * @return the processed data */ - ProcessedData process(ImportContext context); + ImportPreviewResponse process(ImportContext context); /** * Retrieves the name of the Workflow for logging display purposes. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/ImportStatusService.java b/src/main/java/org/breedinginsight/brapps/importer/services/ImportStatusService.java index e1dcefc63..d69934ed0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/ImportStatusService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/ImportStatusService.java @@ -42,17 +42,20 @@ public ImportStatusService(ImportDAO importDAO, ObjectMapper objMapper) { } public void updateMessage(ImportUpload upload, String message) { + log.debug(message); upload.getProgress().setMessage(message); importDAO.update(upload); } public void startUpload(ImportUpload upload, long numberObjects, String message) { + log.debug(message); upload.getProgress().setTotal(numberObjects); upload.getProgress().setMessage(message); importDAO.update(upload); } public void finishUpload(ImportUpload upload, long numberObjects, String message) { + log.debug(message); // Update progress to reflect final finished and inProgress counts. upload.updateProgress(Math.toIntExact(numberObjects), 0); upload.getProgress().setMessage(message); @@ -61,6 +64,7 @@ public void finishUpload(ImportUpload upload, long numberObjects, String message } public void updateMappedData(ImportUpload upload, ImportPreviewResponse response, String message) { + log.debug(message); // Save our results to the db JSON config = new JSON(); String json = config.getGson().toJson(response); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorData.java index 41f432f72..1a4a806b9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorData.java @@ -26,27 +26,27 @@ public class ProcessorData { - static int getNumNewObjects(Map> objectsByName) { + public static int getNumNewObjects(Map> objectsByName) { long numNewObjects = objectsByName.values().stream() .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW) .count(); return Math.toIntExact(numNewObjects); } - static int getNumExistingObjects(Map> objectsByName) { + public static int getNumExistingObjects(Map> objectsByName) { long numExistingObjects = objectsByName.values().stream() .filter(preview -> preview != null && preview.getState() == ImportObjectState.EXISTING) .count(); return Math.toIntExact(numExistingObjects); } - static List getNewObjects(Map> objectsByName) { + public static List getNewObjects(Map> objectsByName) { return objectsByName.values().stream() .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW) .map(preview -> preview.getBrAPIObject()) .collect(Collectors.toList()); } - static Map getMutationsByObjectId(Map> objectsByName, Function dbIdFilter) { + public static Map getMutationsByObjectId(Map> objectsByName, Function dbIdFilter) { return objectsByName.entrySet().stream() .filter(entry -> ImportObjectState.MUTATED == entry.getValue().getState()) .collect(Collectors diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 9983b1aaa..8092a01e9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -16,6 +16,8 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang3.StringUtils; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; @@ -30,6 +32,10 @@ public class ExperimentUtilities { public static final String TIMESTAMP_PREFIX = "TS:"; public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; public static final String MIDNIGHT = "T00:00:00-00:00"; + public static final String MULTIPLE_EXP_TITLES = "File contains more than one Experiment Title"; + public static final String PREEXISTING_EXPERIMENT_TITLE = "Experiment Title already exists"; + + public static List importRowsToExperimentObservations(List importRows) { return importRows.stream() @@ -46,4 +52,11 @@ public static boolean validDateTimeValue(String value) { } return true; } + + public static String getObservationHash(String observationUnitName, String variableName, String studyName) { + String concat = DigestUtils.sha256Hex(observationUnitName) + + DigestUtils.sha256Hex(variableName) + + DigestUtils.sha256Hex(StringUtils.defaultString(studyName)); + return DigestUtils.sha256Hex(concat); + } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java index e05b1cf25..3fe0ba4ee 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; +package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow.steps; // TODO: think this would be for other workflow only public class PopulateModifiedPendingImportObjectsStep { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java index 213411078..96c54712b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -21,6 +21,7 @@ import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.model.ProgramLocation; @@ -40,4 +41,6 @@ public class PendingData { private Map> locationByName; private Map> obsVarDatasetByName; private Map> existingGermplasmByGID; + // TODO: see if we can change this to match PendingImport<> + private Map existingObsByObsHash; } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 4666ac0f1..e59226b81 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -18,16 +18,32 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; import org.breedinginsight.brapps.importer.model.workflow.Workflow; -import org.breedinginsight.brapps.importer.services.pipeline.Pipeline; +import org.breedinginsight.brapps.importer.services.ImportStatusService; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.CommitPendingImportObjectsStep; import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.PopulateExistingPendingImportObjectsStep; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.ProcessStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.PopulateNewPendingImportObjectsStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentPhenotypeService; import javax.inject.Inject; import javax.inject.Named; -import javax.inject.Provider; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; /** * This class represents a workflow for creating a new experiment. The bean name must match the appropriate bean column @@ -35,28 +51,65 @@ */ @Prototype +@Slf4j @Named("CreateNewExperimentWorkflow") public class CreateNewExperimentWorkflow implements Workflow { - private final Provider getExistingStepProvider; - private final Provider processStepProvider; + private final PopulateExistingPendingImportObjectsStep populateExistingPendingImportObjectsStep; + private final PopulateNewPendingImportObjectsStep populateNewPendingImportObjectsStep; + private final CommitPendingImportObjectsStep commitPendingImportObjectsStep; + private final ImportStatusService statusService; + private final ExperimentPhenotypeService experimentPhenotypeService; @Inject - public CreateNewExperimentWorkflow(Provider getExistingStepProvider, - Provider processStepProvider) { - this.getExistingStepProvider = getExistingStepProvider; - this.processStepProvider = processStepProvider; + public CreateNewExperimentWorkflow(PopulateExistingPendingImportObjectsStep populateExistingPendingImportObjectsStep, + PopulateNewPendingImportObjectsStep populateNewPendingImportObjectsStep, + CommitPendingImportObjectsStep commitPendingImportObjectsStep, + ImportStatusService statusService, + ExperimentPhenotypeService experimentPhenotypeService) { + this.populateExistingPendingImportObjectsStep = populateExistingPendingImportObjectsStep; + this.populateNewPendingImportObjectsStep = populateNewPendingImportObjectsStep; + this.commitPendingImportObjectsStep = commitPendingImportObjectsStep; + this.statusService = statusService; + this.experimentPhenotypeService = experimentPhenotypeService; } @Override - public ProcessedData process(ImportContext context) { - // TODO - Pipeline pipeline = new Pipeline<>(getExistingStepProvider.get()) - .addProcessingStep(processStepProvider.get()); - ProcessedData processed = pipeline.execute(context); - - // TODO: return actual data - return processed; + public ImportPreviewResponse process(ImportContext context) { + + ImportUpload upload = context.getUpload(); + boolean commit = context.isCommit(); + + // Make sure the file does not contain obs unit ids before proceeding + if (containsObsUnitIDs(context)) { + // TODO: get file name + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Error detected in file, XXX.xls. ObsUnitIDs are detected. Import cannot proceed"); + } + + statusService.updateMessage(upload, "Checking existing experiment objects in brapi service and mapping data"); + + ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(context); + ProcessContext processContext = populateExistingPendingImportObjectsStep.process(context, phenotypeData); + ProcessedData processedData = populateNewPendingImportObjectsStep.process(processContext, phenotypeData); + ImportPreviewResponse response = buildImportPreviewResponse(processedData, upload); + + statusService.updateMappedData(upload, response, "Finished mapping data to brapi objects"); + + // preview data + if (!commit) { + statusService.updateOk(upload); + return response; + } + + // commit data + long totalObjects = getNewObjectCount(processedData); + statusService.startUpload(upload, totalObjects, "Starting upload to brapi service"); + statusService.updateMessage(upload, "Creating new experiment objects in brapi service"); + + commitPendingImportObjectsStep.process(processContext, processedData); + + statusService.finishUpload(upload, totalObjects, "Completed upload to brapi service"); + return response; } /** @@ -69,5 +122,38 @@ public ProcessedData process(ImportContext context) { public String getName() { return "CreateNewExperimentWorkflow"; } + + // TODO: move to shared area + private ImportPreviewResponse buildImportPreviewResponse(ProcessedData processedData, + ImportUpload upload) { + Map statistics = processedData.getStatistics(); + Map mappedBrAPIImport = processedData.getMappedBrAPIImport(); + + ImportPreviewResponse response = new ImportPreviewResponse(); + response.setStatistics(statistics); + List mappedBrAPIImportList = new ArrayList<>(mappedBrAPIImport.values()); + response.setRows(mappedBrAPIImportList); + response.setDynamicColumnNames(upload.getDynamicColumnNamesList()); + return response; + } + + // TODO: move to shared area + private long getNewObjectCount(ProcessedData processedData) { + // get total number of new brapi objects to create + long totalObjects = 0; + for (ImportPreviewStatistics stats : processedData.getStatistics().values()) { + totalObjects += stats.getNewObjectCount(); + } + return totalObjects; + } + + private boolean containsObsUnitIDs(ImportContext importContext) { + List importRows = importContext.getImportRows(); + return importRows.stream() + .anyMatch(row -> { + ExperimentObservation expRow = (ExperimentObservation) row; + return StringUtils.isNotBlank(expRow.getObsUnitID()); + }); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java index ea839298b..836ae2731 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; +import io.micronaut.context.annotation.Property; import org.apache.commons.lang3.StringUtils; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; @@ -18,7 +19,13 @@ import java.util.UUID; import java.util.function.Supplier; -public class CreatePendingImportPopulator implements PendingImportObjectPopulator { +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.MULTIPLE_EXP_TITLES; +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.PREEXISTING_EXPERIMENT_TITLE; + +public class CreatePendingImportPopulator implements PendingImportObjectPopulator { + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; @Override public PendingImportObject populateTrial(ImportContext importContext, @@ -54,14 +61,20 @@ public PendingImportObject populateTrial(ImportContext importContext } BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); - // TODO: move + // NOTE: moved up a level //trialByNameNoScope.put(importRow.getExpTitle(), trialPio); } return trialPio; } + @Override + public PendingImportObject populateStudy(ImportContext importContext, + Supplier expSeqValue, + ExperimentObservation importRow, + Supplier envNextVal) { + return null; + } - } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index 02ab60e3e..44c194cbf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -16,10 +16,62 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.request.BrAPIListNewRequest; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; -import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; -public class CommitPendingImportDataStep implements ProcessingStep { - +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Slf4j +public class CommitPendingImportObjectsStep { + + public void process(ProcessContext processContext, ProcessedData processedData) { + + PendingData pendingData = processContext.getPendingData(); + + List newTrials = ProcessorData.getNewObjects(pendingData.getTrialByNameNoScope()); + + List newLocations = ProcessorData.getNewObjects(pendingData.getLocationByName()) + .stream() + .map(location -> ProgramLocationRequest.builder() + .name(location.getName()) + .build()) + .collect(Collectors.toList()); + + List newStudies = ProcessorData.getNewObjects(pendingData.getStudyByNameNoScope()); + + List newDatasetRequests = ProcessorData.getNewObjects(pendingData.getObsVarDatasetByName()).stream().map(details -> { + BrAPIListNewRequest request = new BrAPIListNewRequest(); + request.setListName(details.getListName()); + request.setListType(details.getListType()); + request.setExternalReferences(details.getExternalReferences()); + request.setAdditionalInfo(details.getAdditionalInfo()); + request.data(details.getData()); + return request; + }).collect(Collectors.toList()); + + Map datasetNewDataById = ProcessorData + .getMutationsByObjectId(pendingData.getObsVarDatasetByName(), BrAPIListSummary::getListDbId); + + List newObservationUnits = ProcessorData.getNewObjects(pendingData.getObservationUnitByNameNoScope()); + + // filter out observations with no 'value' so they will not be saved + List newObservations = ProcessorData.getNewObjects(this.observationByHash) + .stream() + .filter(obs -> !obs.getValue().isBlank()) + .collect(Collectors.toList()); + + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index 8ee355dc3..d3f83d45b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -17,7 +17,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; import io.micronaut.context.annotation.Property; -import io.micronaut.context.annotation.Prototype; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -29,6 +28,7 @@ import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.*; @@ -37,18 +37,20 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedStudyService; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedTrialService; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentStudyService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentTrialService; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.model.Trait; import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; +import javax.inject.Singleton; import java.util.*; import java.util.stream.Collectors; @@ -57,9 +59,9 @@ * steps rather than another layer of services. */ -@Prototype +@Singleton @Slf4j -public class PopulateExistingPendingImportObjectsStep implements ProcessingStep { +public class PopulateExistingPendingImportObjectsStep { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final BrAPITrialDAO brAPITrialDAO; @@ -67,8 +69,9 @@ public class PopulateExistingPendingImportObjectsStep implements ProcessingStep< private final ProgramLocationService locationService; private final BrAPIListDAO brAPIListDAO; private final BrAPIGermplasmDAO brAPIGermplasmDAO; - private final SharedStudyService sharedStudyService; - private final SharedTrialService sharedTrialService; + private final BrAPIObservationDAO brAPIObservationDAO; + private final ExperimentStudyService experimentStudyService; + private final ExperimentTrialService experimentTrialService; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @@ -80,32 +83,34 @@ public PopulateExistingPendingImportObjectsStep(BrAPIObservationUnitDAO brAPIObs ProgramLocationService locationService, BrAPIListDAO brAPIListDAO, BrAPIGermplasmDAO brAPIGermplasmDAO, - SharedStudyService sharedStudyService, - SharedTrialService sharedTrialService) { + BrAPIObservationDAO brAPIObservationDAO, + ExperimentStudyService experimentStudyService, + ExperimentTrialService experimentTrialService) { this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.brAPITrialDAO = brAPITrialDAO; this.brAPIStudyDAO = brAPIStudyDAO; this.locationService = locationService; this.brAPIListDAO = brAPIListDAO; this.brAPIGermplasmDAO = brAPIGermplasmDAO; - this.sharedStudyService = sharedStudyService; - this.sharedTrialService = sharedTrialService; + this.brAPIObservationDAO = brAPIObservationDAO; + this.experimentStudyService = experimentStudyService; + this.experimentTrialService = experimentTrialService; } - @Override - public ProcessContext process(ImportContext input) { + public ProcessContext process(ImportContext input, ProcessedPhenotypeData phenotypeData) { List experimentImportRows = ExperimentUtilities.importRowsToExperimentObservations(input.getImportRows()); Program program = input.getProgram(); // Populate pending objects with existing status Map> observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); - Map> trialByNameNoScope = sharedTrialService.initializeTrialByNameNoScope(program, observationUnitByNameNoScope, experimentImportRows); + Map> trialByNameNoScope = experimentTrialService.initializeTrialByNameNoScope(program, observationUnitByNameNoScope, experimentImportRows); Map> studyByNameNoScope = initializeStudyByNameNoScope(program, trialByNameNoScope, observationUnitByNameNoScope, experimentImportRows); // interesting we're using our data model instead of brapi for locations Map> locationByName = initializeUniqueLocationNames(program, studyByNameNoScope, experimentImportRows); Map> obsVarDatasetByName = initializeObsVarDatasetByName(program, trialByNameNoScope, experimentImportRows); Map> existingGermplasmByGID = initializeExistingGermplasmByGID(program, observationUnitByNameNoScope, experimentImportRows); + Map existingObsByObsHash = fetchExistingObservations(phenotypeData.getReferencedTraits(), studyByNameNoScope, program); PendingData existing = PendingData.builder() .observationUnitByNameNoScope(observationUnitByNameNoScope) @@ -114,6 +119,7 @@ public ProcessContext process(ImportContext input) { .locationByName(locationByName) .obsVarDatasetByName(obsVarDatasetByName) .existingGermplasmByGID(existingGermplasmByGID) + .existingObsByObsHash(existingObsByObsHash) .build(); return ProcessContext.builder() @@ -253,7 +259,7 @@ private Map> initializeStudyByNameNoScop UUID experimentId = trial.get().getId(); existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); for (BrAPIStudy existingStudy : existingStudies) { - sharedStudyService.processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); + experimentStudyService.processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); } } catch (ApiException e) { log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); @@ -302,9 +308,9 @@ private void initializeStudiesForExistingObservationUnits( .getStudyDbId()) .collect(Collectors.toSet()); - List studies = sharedStudyService.fetchStudiesByDbId(studyDbIds, program); + List studies = experimentStudyService.fetchStudiesByDbId(studyDbIds, program); for (BrAPIStudy study : studies) { - sharedStudyService.processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); + experimentStudyService.processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); } } @@ -487,4 +493,63 @@ private ArrayList getGermplasmByAccessionNumber( return resultGermplasm; } + /** + * Fetches existing observations based on the given referenced traits, studyByNameNoScope map, and program. + * + * @param referencedTraits The list of referenced traits. + * @param studyByNameNoScope The map of studies by name without scope. + * @param program The program. + * @return A map of existing observations with their unique keys. + */ + private Map fetchExistingObservations(List referencedTraits, + Map> studyByNameNoScope, + Program program) { + Set ouDbIds = new HashSet<>(); + Set variableDbIds = new HashSet<>(); + Map variableNameByDbId = new HashMap<>(); + Map ouNameByDbId = new HashMap<>(); + Map studyNameByDbId = studyByNameNoScope.values() + .stream() + .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) + .map(PendingImportObject::getBrAPIObject) + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); + + studyNameByDbId.keySet().forEach(studyDbId -> { + try { + brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { + if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { + ouDbIds.add(ou.getObservationUnitDbId()); + } + ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + }); + } catch (ApiException e) { + throw new RuntimeException(e); + } + }); + + for (Trait referencedTrait : referencedTraits) { + variableDbIds.add(referencedTrait.getObservationVariableDbId()); + variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); + } + + List existingObservations = new ArrayList<>(); + try { + existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); + } catch (ApiException e) { + throw new RuntimeException(e); + } + + return existingObservations.stream() + .map(obs -> { + String studyName = studyNameByDbId.get(obs.getStudyDbId()); + String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); + String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); + + String key = ExperimentUtilities.getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); + + return Map.entry(key, obs); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index 13870fc51..cd4faf164 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -1,4 +1,601 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingImportObjectData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.PendingImportObjectPopulator; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentSeasonService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentValidateService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.model.User; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; +import org.jooq.DSLContext; +import tech.tablesaw.api.Table; +import org.breedinginsight.model.Trait; +import tech.tablesaw.columns.Column; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.math.BigInteger; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.*; + +@Singleton +@Slf4j public class PopulateNewPendingImportObjectsStep { -} + + private final ExperimentValidateService experimentValidateService; + private final ExperimentSeasonService experimentSeasonService; + private final BrAPIObservationDAO brAPIObservationDAO; + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private final DSLContext dsl; + + @Inject + public PopulateNewPendingImportObjectsStep(ExperimentValidateService experimentValidateService, + ExperimentSeasonService experimentSeasonService, + BrAPIObservationDAO brAPIObservationDAO, + BrAPIObservationUnitDAO brAPIObservationUnitDAO, + DSLContext dsl) { + this.experimentValidateService = experimentValidateService; + this.experimentSeasonService = experimentSeasonService; + this.brAPIObservationDAO = brAPIObservationDAO; + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + this.dsl = dsl; + } + + public ProcessedData process(ProcessContext context, ProcessedPhenotypeData phenotypeData) { + + Table data = context.getImportContext().getData(); + ImportUpload upload = context.getImportContext().getUpload(); + ImportContext importContext = context.getImportContext(); + + + + // TODO: implement + return new ProcessedData(); + } + + + // initNew + // TODO: move to shared service + private void populatePendingImportObjects(ImportContext importContext, + ProcessedPhenotypeData phenotypeData, + PendingImportObjectPopulator pioPopulator) { + + List importRows = importContext.getImportRows(); + Program program = importContext.getProgram(); + boolean commit = importContext.isCommit(); + + Supplier expNextVal = getNextExperimentSequenceNumber(program); + Supplier envNextVal = getNextEnvironmentSequenceNumber(program); + + // NOTE: this was moved to the get existing step and kept in PendingData + // existingObsByObsHash = fetchExistingObservations(referencedTraits, program); + + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + + populateIndependentVariablePIOsForRow(); + + // ... (Common logic from the original method) + + PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); + + processObservations(importContext, phenotypeData, importRow, rowNum, commit, importRow.getEnvYear(), obsUnitPIO, studyPIO); + } + } + + + /** + * Returns a Supplier that generates the next experiment sequence number based on the given Program. + * + * @param program the Program for which to generate the next experiment sequence number + * @return a Supplier that generates the next experiment sequence number + * @throws HttpStatusException if the program is not properly configured for observation unit import + */ + private Supplier getNextExperimentSequenceNumber(Program program) { + String expSequenceName = program.getExpSequence(); + if (expSequenceName == null) { + log.error(String.format("Program, %s, is missing a value in the exp sequence column.", program.getName())); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for observation unit import"); + } + return () -> dsl.nextval(expSequenceName.toLowerCase()); + } + + /** + * Retrieves the next environment sequence number for a given program. + * + * @param program The program for which to get the next environment sequence number. + * @return A Supplier representing a function that generates the next environment sequence number. + * @throws HttpStatusException If the program is not properly configured for environment import. + */ + private Supplier getNextEnvironmentSequenceNumber(Program program) { + String envSequenceName = program.getEnvSequence(); + if (envSequenceName == null) { + log.error(String.format("Program, %s, is missing a value in the env sequence column.", program.getName())); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for environment import"); + } + return () -> dsl.nextval(envSequenceName.toLowerCase()); + } + + /** + * Populates independent variable PendingImportObjectData for a given row of import data. + * + * @param importContext The import context. + * @param phenotypeData The processed phenotype data. + * @param pendingData The pending data. + * @param importRow The import row. + * @param expNextVal The supplier for generating experiment next value. + * @param envNextVal The supplier for generating environment next value. + * @param pioPopulator The pending import object populator. + * @return The populated independent variable PendingImportObjectData. + * @throws MissingRequiredInfoException If any required information is missing. + * @throws UnprocessableEntityException If the entity is unprocessable. + * @throws ApiException If there is an API exception. + */ + private PendingImportObjectData populateIndependentVariablePIOsForRow(ImportContext importContext, + ProcessedPhenotypeData phenotypeData, + PendingData pendingData, + ExperimentObservation importRow, + Supplier expNextVal, + Supplier envNextVal, + PendingImportObjectPopulator pioPopulator) + throws MissingRequiredInfoException, UnprocessableEntityException, ApiException { + + Program program = importContext.getProgram(); + User user = importContext.getUser(); + boolean commit = importContext.isCommit(); + List referencedTraits = phenotypeData.getReferencedTraits(); + + PendingImportObject trialPIO = null; + try { + trialPIO = pioPopulator.populateTrial(importContext, pendingData, importRow, expNextVal); + + // moved up a level + if (trialPIO.getState() == ImportObjectState.NEW) { + pendingData.getTrialByNameNoScope().put(importRow.getExpTitle(), trialPIO); + } + } catch (UnprocessableEntityException e) { + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } + + String expSeqValue = null; + if (commit) { + expSeqValue = trialPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString(); + } + + if (commit) { + fetchOrCreateDatasetPIO(importRow, program, referencedTraits); + } + + fetchOrCreateLocationPIO(importRow); + + PendingImportObject studyPIO = fetchOrCreateStudyPIO(program, commit, expSeqValue, importRow, envNextVal); + + String envSeqValue = null; + if (commit) { + envSeqValue = studyPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER) + .getAsString(); + } + + PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); + + return PendingImportObjectData.builder() + .trialPIO(trialPIO) + .studyPIO(studyPIO) + .obsUnitPIO(obsUnitPIO) + .build(); + } + + private void processObservations(ImportContext importContext, ProcessedPhenotypeData phenotypeData, ExperimentObservation importRow, + int rowNum, boolean commit, String studyYear, + PendingImportObject obsUnitPIO, + PendingImportObject studyPIO) + throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { + Program program = importContext.getProgram(); + User user = importContext.getUser(); + List> phenotypeCols = phenotypeData.getPhenotypeCols(); + Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); + List referencedTraits = phenotypeData.getReferencedTraits(); + + for (Column column : phenotypeCols) { + // ... (Logic for processing observations) + } + } + + // TODO: move common code out + private void initNewBrapiData(ImportContext importContext, ProcessedPhenotypeData phenotypeData) + throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { + + Program program = importContext.getProgram(); + List importRows = importContext.getImportRows(); + User user = importContext.getUser(); + boolean commit = importContext.isCommit(); + Map existingObsByObsHash; + + List referencedTraits = phenotypeData.getReferencedTraits(); + List> phenotypeCols = phenotypeData.getPhenotypeCols(); + Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); + + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + + + for (Column column : phenotypeCols) { + //If associated timestamp column, add + String dateTimeValue = null; + if (timeStampColByPheno.containsKey(column.name())) { + dateTimeValue = timeStampColByPheno.get(column.name()).getString(rowNum); + //If no timestamp, set to midnight + if (!dateTimeValue.isBlank() && !validDateTimeValue(dateTimeValue)) { + dateTimeValue += MIDNIGHT; + } + } + + // get the study year either referenced from the observation unit or listed explicitly on the import row + // TODO: handle this different workflows + String studyYear = hasAllReferenceUnitIds ? studyPIO.getBrAPIObject().getSeasons().get(0) : importRow.getEnvYear(); + String seasonDbId = experimentSeasonService.yearToSeasonDbId(studyYear, program.getId()); + fetchOrCreateObservationPIO( + program, + user, + importRow, + column, //column.name() gets phenotype name + rowNum, + dateTimeValue, + commit, + seasonDbId, + obsUnitPIO, + studyPIO, + referencedTraits + ); + } + } + } + + private Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { + Set ouDbIds = new HashSet<>(); + Set variableDbIds = new HashSet<>(); + Map variableNameByDbId = new HashMap<>(); + Map ouNameByDbId = new HashMap<>(); + Map studyNameByDbId = studyByNameNoScope.values() + .stream() + .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) + .map(PendingImportObject::getBrAPIObject) + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); + + studyNameByDbId.keySet().forEach(studyDbId -> { + try { + brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { + if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { + ouDbIds.add(ou.getObservationUnitDbId()); + } + ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + }); + } catch (ApiException e) { + throw new RuntimeException(e); + } + }); + + for (Trait referencedTrait : referencedTraits) { + variableDbIds.add(referencedTrait.getObservationVariableDbId()); + variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); + } + + List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); + + return existingObservations.stream() + .map(obs -> { + String studyName = studyNameByDbId.get(obs.getStudyDbId()); + String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); + String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); + + String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); + + return Map.entry(key, obs); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private PendingImportObject fetchOrCreateTrialPIO( + Program program, + User user, + boolean commit, + ExperimentObservation importRow, + Supplier expNextVal + ) throws UnprocessableEntityException { + PendingImportObject trialPio; + + // use the prior trial if observation unit IDs are supplied + // TODO: handle multiple workflows + if (hasAllReferenceUnitIds) { + trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); + + // otherwise create a new trial, but there can be only one allowed + } else { + if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { + PendingImportObject envPio; + trialPio = trialByNameNoScope.get(importRow.getExpTitle()); + envPio = studyByNameNoScope.get(importRow.getEnv()); + + // creating new units for existing experiments and environments is not possible + if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && + (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ + throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); + } + } else if (!trialByNameNoScope.isEmpty()) { + throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); + } else { + UUID id = UUID.randomUUID(); + String expSeqValue = null; + if (commit) { + expSeqValue = expNextVal.get().toString(); + } + BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); + trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); + trialByNameNoScope.put(importRow.getExpTitle(), trialPio); + } + } + return trialPio; + } + + private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) throws UnprocessableEntityException { + PendingImportObject pio; + + // TODO: multiple workflows + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + String name = String.format("Observation Dataset [%s-%s]", + program.getKey(), + trialPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString()); + if (obsVarDatasetByName.containsKey(name)) { + pio = obsVarDatasetByName.get(name); + } else { + UUID id = UUID.randomUUID(); + BrAPIListDetails newDataset = importRow.constructDatasetDetails( + name, + id, + BRAPI_REFERENCE_SOURCE, + program, + trialPIO.getId().toString()); + pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); + trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); + if (ImportObjectState.EXISTING == trialPIO.getState()) { + trialPIO.setState(ImportObjectState.MUTATED); + } + obsVarDatasetByName.put(name, pio); + } + addObsVarsToDatasetDetails(pio, referencedTraits, program); + } + + private void fetchOrCreateLocationPIO(ExperimentObservation importRow) { + PendingImportObject pio; + // TODO: multiple workflows + String envLocationName = hasAllReferenceUnitIds ? + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName() : importRow.getEnvLocation(); + if (!locationByName.containsKey((importRow.getEnvLocation()))) { + ProgramLocation newLocation = new ProgramLocation(); + newLocation.setName(envLocationName); + pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); + this.locationByName.put(envLocationName, pio); + } + } + + private PendingImportObject fetchOrCreateStudyPIO( + Program program, + boolean commit, + String expSequenceValue, + ExperimentObservation importRow, + Supplier envNextVal + ) throws UnprocessableEntityException { + PendingImportObject pio; + // TODO: multiple workflows + if (hasAllReferenceUnitIds) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), + program.getKey() + ); + pio = studyByNameNoScope.get(studyName); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { + pio = studyByNameNoScope.get(importRow.getEnv()); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else { + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + UUID trialID = trialPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); + newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies + + // It is assumed that the study has only one season, And that the Years and not + // the dbId's are stored in getSeason() list. + String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season + if (commit) { + if(StringUtils.isNotBlank(year)) { + String seasonID = this.yearToSeasonDbId(year, program.getId()); + newStudy.setSeasons(Collections.singletonList(seasonID)); + } + } else { + addYearToStudyAdditionalInfo(program, newStudy, year); + } + + pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); + this.studyByNameNoScope.put(importRow.getEnv(), pio); + } + return pio; + } + + private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { + PendingImportObject pio; + String key = createObservationUnitKey(importRow); + // TODO: multiple workflows + if (hasAllReferenceUnitIds) { + pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); + } else if (observationUnitByNameNoScope.containsKey(key)) { + pio = observationUnitByNameNoScope.get(key); + } else { + String germplasmName = ""; + if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { + germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) + .getBrAPIObject() + .getGermplasmName(); + } + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; + UUID trialID = trialPIO.getId(); + UUID datasetId = null; + if (commit) { + datasetId = UUID.fromString(trialPIO.getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + } + PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); + + // check for existing units if this is an existing study + if (studyPIO.getBrAPIObject().getStudyDbId() != null) { + List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); + List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); + if (matchingOU.isEmpty()) { + throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); + } else { + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); + } + } else { + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); + } + this.observationUnitByNameNoScope.put(key, pio); + } + return pio; + } + + private void fetchOrCreateObservationPIO(Program program, + User user, + ExperimentObservation importRow, + Column column, + Integer rowNum, + String timeStampValue, + boolean commit, + String seasonDbId, + PendingImportObject obsUnitPIO, + PendingImportObject studyPIO, + List referencedTraits) throws ApiException, UnprocessableEntityException { + PendingImportObject pio; + BrAPIObservation newObservation; + String variableName = column.name(); + String value = column.getString(rowNum); + String key; + + // TODO: multiple workflows + if (hasAllReferenceUnitIds) { + String unitName = obsUnitPIO.getBrAPIObject().getObservationUnitName(); + String studyName = studyPIO.getBrAPIObject().getStudyName(); + key = getObservationHash(studyName + unitName, variableName, studyName); + } else { + key = getImportObservationHash(importRow, variableName); + } + + if (existingObsByObsHash.containsKey(key)) { + if (!isObservationMatched(key, value, column, rowNum)){ + + // prior observation with updated value + newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); + if (!isValueMatched(key, value)){ + newObservation.setValue(value); + } else if (!StringUtils.isBlank(timeStampValue) && !isTimestampMatched(key, timeStampValue)) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timeStampValue)); + newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + } + pio = new PendingImportObject<>(ImportObjectState.MUTATED, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); + } else { + + // prior observation + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(existingObsByObsHash.get(key), BrAPIObservation.class, program)); + } + + observationByHash.put(key, pio); + } else if (!this.observationByHash.containsKey(key)){ + + // new observation + // TODO: multiple workflows + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + + UUID trialID = trialPIO.getId(); + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject(), commit, program, user, BRAPI_REFERENCE_SOURCE, trialID, studyID, obsUnitPIO.getId(), id); + //NOTE: Can't parse invalid timestamp value, so have to skip if invalid. + // Validation error should be thrown for offending value, but that doesn't happen until later downstream + if (timeStampValue != null && !timeStampValue.isBlank() && (validDateValue(timeStampValue) || validDateTimeValue(timeStampValue))) { + newObservation.setObservationTimeStamp(OffsetDateTime.parse(timeStampValue)); + } + + newObservation.setStudyDbId(studyPIO.getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations + + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); + this.observationByHash.put(key, pio); + } + } + + +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java index b6f6e7796..cc81a8498 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java @@ -20,18 +20,14 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -43,64 +39,55 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; import org.breedinginsight.brapps.importer.services.processors.experiment.services.PendingImportObjectPopulator; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedPhenotypeService; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedSeasonService; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedValidateService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentPhenotypeService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentSeasonService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentValidateService; import org.breedinginsight.model.Program; -import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.model.User; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; import tech.tablesaw.api.Table; import org.breedinginsight.model.Trait; -import tech.tablesaw.columns.Column; import javax.inject.Inject; import java.math.BigInteger; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; import java.util.*; import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.*; @Prototype @Slf4j -public class ProcessStep implements ProcessingStep { +public class ProcessStep implements { - private final SharedValidateService sharedValidateService; - private final SharedSeasonService sharedSeasonService; - private final SharedPhenotypeService sharedPhenotypeService; + private final ExperimentValidateService experimentValidateService; + private final ExperimentSeasonService experimentSeasonService; + private final ExperimentPhenotypeService experimentPhenotypeService; private final BrAPIObservationDAO brAPIObservationDAO; private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final DSLContext dsl; @Inject - public ProcessStep(SharedValidateService sharedValidateService, - SharedSeasonService sharedSeasonService, - SharedPhenotypeService sharedPhenotypeService, + public ProcessStep(ExperimentValidateService experimentValidateService, + ExperimentSeasonService experimentSeasonService, + ExperimentPhenotypeService experimentPhenotypeService, BrAPIObservationDAO brAPIObservationDAO, BrAPIObservationUnitDAO brAPIObservationUnitDAO, DSLContext dsl) { - this.sharedValidateService = sharedValidateService; - this.sharedSeasonService = sharedSeasonService; - this.sharedPhenotypeService = sharedPhenotypeService; + this.experimentValidateService = experimentValidateService; + this.experimentSeasonService = experimentSeasonService; + this.experimentPhenotypeService = experimentPhenotypeService; this.brAPIObservationDAO = brAPIObservationDAO; this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.dsl = dsl; } - @Override public ProcessedData process(ProcessContext context) { Table data = context.getImportContext().getData(); ImportUpload upload = context.getImportContext().getUpload(); ImportContext importContext = context.getImportContext(); - ProcessedPhenotypeData phenotypeData = sharedPhenotypeService.extractPhenotypes(importContext); + ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(importContext); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPendingImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPendingImportService.java index a5d091d69..9a1af1c37 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPendingImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPendingImportService.java @@ -16,7 +16,7 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment.services; -public class SharedPendingImportService { +public class ExperimentPendingImportService { } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPhenotypeService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPhenotypeService.java index 6b9bd7dc5..ce744234d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPhenotypeService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPhenotypeService.java @@ -37,13 +37,13 @@ @Singleton @Slf4j -public class SharedPhenotypeService { +public class ExperimentPhenotypeService { - private final SharedValidateService sharedValidateService; + private final ExperimentValidateService experimentValidateService; @Inject - public SharedPhenotypeService(SharedValidateService sharedValidateService) { - this.sharedValidateService = sharedValidateService; + public ExperimentPhenotypeService(ExperimentValidateService experimentValidateService) { + this.experimentValidateService = experimentValidateService; } /** @@ -58,7 +58,7 @@ public ProcessedPhenotypeData extractPhenotypes(ImportContext importContext) { Program program = importContext.getProgram(); DynamicColumnParser.DynamicColumnParseResult result = DynamicColumnParser.parse(data, upload.getDynamicColumnNames()); - List traits = sharedValidateService.verifyTraits(program.getId(), result); + List traits = experimentValidateService.verifyTraits(program.getId(), result); Map> timeStampColByPheno = new HashMap<>(); //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java index d168a8a16..5fd67a023 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java @@ -32,7 +32,7 @@ @Singleton @Slf4j -public class SharedSeasonService { +public class ExperimentSeasonService { private final BrAPISeasonDAO brAPISeasonDAO; @@ -40,7 +40,7 @@ public class SharedSeasonService { private Map yearToSeasonDbIdCache = new HashMap<>(); @Inject - public SharedSeasonService(BrAPISeasonDAO brAPISeasonDAO) { + public ExperimentSeasonService(BrAPISeasonDAO brAPISeasonDAO) { this.brAPISeasonDAO = brAPISeasonDAO; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentStudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentStudyService.java index 0e9aef883..c1bd4cd2c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentStudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentStudyService.java @@ -40,7 +40,7 @@ @Singleton @Slf4j -public class SharedStudyService { +public class ExperimentStudyService { private final BrAPISeasonDAO brAPISeasonDAO; private final BrAPIStudyDAO brAPIStudyDAO; @@ -49,8 +49,8 @@ public class SharedStudyService { private String BRAPI_REFERENCE_SOURCE; @Inject - public SharedStudyService(BrAPISeasonDAO brAPISeasonDAO, - BrAPIStudyDAO brAPIStudyDAO) { + public ExperimentStudyService(BrAPISeasonDAO brAPISeasonDAO, + BrAPIStudyDAO brAPIStudyDAO) { this.brAPISeasonDAO = brAPISeasonDAO; this.brAPIStudyDAO = brAPIStudyDAO; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java index d3d5efd1f..9db927345 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java @@ -41,17 +41,17 @@ @Singleton @Slf4j -public class SharedTrialService { +public class ExperimentTrialService { private final BrAPITrialDAO brAPITrialDAO; - private final SharedStudyService studyService; + private final ExperimentStudyService studyService; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @Inject - public SharedTrialService(BrAPITrialDAO brAPITrialDAO, - SharedStudyService studyService) { + public ExperimentTrialService(BrAPITrialDAO brAPITrialDAO, + ExperimentStudyService studyService) { this.brAPITrialDAO = brAPITrialDAO; this.studyService = studyService; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java index f2edfc3f5..af2bcb29e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java @@ -42,13 +42,13 @@ @Singleton @Slf4j -public class SharedValidateService { +public class ExperimentValidateService { private final OntologyService ontologyService; private final FileMappingUtil fileMappingUtil; @Inject - public SharedValidateService(OntologyService ontologyService, FileMappingUtil fileMappingUtil) { + public ExperimentValidateService(OntologyService ontologyService, FileMappingUtil fileMappingUtil) { this.ontologyService = ontologyService; this.fileMappingUtil = fileMappingUtil; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java index d189a461d..9ca689447 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java @@ -17,6 +17,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.services; +import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -36,6 +37,9 @@ PendingImportObject populateTrial( Supplier expNextVal ) throws UnprocessableEntityException; - - + PendingImportObject populateStudy( + ImportContext importContext, + Supplier expSeqValue, + ExperimentObservation importRow, + Supplier envNextVal); } From be583362b132baae57e3491434fee617b3f26370 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 10 Jun 2024 13:19:54 -0400 Subject: [PATCH 104/203] fix instantiation of gson --- .../appendoverwrite/middleware/process/ImportTableProcess.java | 1 + .../appendoverwrite/middleware/process/InitialData.java | 2 +- .../appendoverwrite/middleware/process/OverwrittenData.java | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index e8413991d..379b74df7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -74,6 +74,7 @@ public ImportTableProcess(StudyService studyService, this.observationService = observationService; this.experimentUtil = experimentUtil; this.gson = new Gson(); + this.fieldValidator = fieldValidator; this.statistic = statistic; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java index 7a19967e8..03a139e89 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java @@ -39,7 +39,6 @@ public class InitialData extends VisitedObservationData { FieldValidator fieldValidator; @Inject StudyService studyService; - @Inject Gson gson; public InitialData(boolean isCommit, @@ -65,6 +64,7 @@ public InitialData(boolean isCommit, this.observationUnit = observationUnit; this.user = user; this.program = program; + this.gson = new Gson(); } @Override public Optional> getValidationErrors() { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java index 9a59da97c..3473151aa 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java @@ -27,7 +27,6 @@ public class OverwrittenData extends VisitedObservationData { @Inject FieldValidator fieldValidator; - @Inject Gson gson; boolean canOverwrite; boolean isCommit; @@ -66,6 +65,7 @@ public OverwrittenData(boolean canOverwrite, this.observation = observation; this.userId = userId; this.program = program; + this.gson = new Gson(); } @Override From bce2c7fbcce36ee87698257b521b9f87ffd2aa95 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:11:36 -0400 Subject: [PATCH 105/203] change Workflow field names --- .../model/workflow/ImportWorkflow.java | 4 ++-- .../ExperimentWorkflowNavigator.java | 18 +++++++++--------- .../AppendOverwritePhenotypesWorkflow.java | 4 ++-- .../workflow/CreateNewExperimentWorkflow.java | 4 ++-- .../workflow/CreateNewEnvironmentWorkflow.java | 4 ++-- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java index b26a886bd..30dbac06a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java @@ -25,7 +25,7 @@ @ToString @AllArgsConstructor public class ImportWorkflow { - private String urlFragment; - private String displayName; + private String id; + private String name; private int order; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index bcf12d543..b43b859f5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -42,22 +42,22 @@ public enum Workflow { APPEND_OVERWRITE("append-dataset", "Append experimental dataset"), APPEND_ENVIRONMENT("append-environment", "Create new experimental environment"); - private String urlFragment; - private String displayName; + private String id; + private String name; - Workflow(String urlFragment, String displayName) { + Workflow(String id, String name) { - this.urlFragment = urlFragment; - this.displayName = displayName; + this.id = id; + this.name = name; } - public String getUrlFragment() { - return urlFragment; + public String getId() { + return id; } - public String getDisplayName() { return displayName; } + public String getName() { return name; } public boolean isEqual(String value) { - return Optional.ofNullable(urlFragment.equals(value)).orElse(false); + return Optional.ofNullable(id.equals(value)).orElse(false); } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index e64a2f88a..6ff074059 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -23,8 +23,8 @@ public AppendOverwritePhenotypesWorkflow(){ public Optional process(ImportServiceContext context) { // Workflow processing the context ImportWorkflow workflow = ImportWorkflow.builder() - .urlFragment(getWorkflow().getUrlFragment()) - .displayName(getWorkflow().getDisplayName()) + .id(getWorkflow().getId()) + .name(getWorkflow().getName()) .build(); // No-preview result diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 022727c5a..503e4e5f0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -23,8 +23,8 @@ public CreateNewExperimentWorkflow(){ public Optional process(ImportServiceContext context) { // Workflow processing the context ImportWorkflow workflow = ImportWorkflow.builder() - .urlFragment(getWorkflow().getUrlFragment()) - .displayName(getWorkflow().getDisplayName()) + .id(getWorkflow().getId()) + .name(getWorkflow().getName()) .build(); // No-preview result diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java index a5695e7f8..1b7eba9ce 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java @@ -23,8 +23,8 @@ public CreateNewEnvironmentWorkflow(){ public Optional process(ImportServiceContext context) { // Workflow processing the context ImportWorkflow workflow = ImportWorkflow.builder() - .urlFragment(getWorkflow().getUrlFragment()) - .displayName(getWorkflow().getDisplayName()) + .id(getWorkflow().getId()) + .name(getWorkflow().getName()) .build(); // No-preview result From 6060fcc990177a9c7e779af2cc66a07937cf33e4 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:29:14 -0400 Subject: [PATCH 106/203] use processorManager until Workflow business logic in place --- .../importer/model/imports/DomainImportService.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 5f89de64a..d826033fb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -21,6 +21,7 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; @@ -35,6 +36,7 @@ @Slf4j public abstract class DomainImportService implements BrAPIImportService { + // TODO: delete processor fields once WorkflowNavigator is used private final Provider experimentProcessorProvider; private final Provider processorManagerProvider; private final Workflow workflowNavigator; @@ -66,7 +68,15 @@ public ImportPreviewResponse process(ImportServiceContext context) log.info("Workflow: " + context.getWorkflow()); } - return workflowNavigator.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); + // TODO: return results from WorkflowNavigator once processing logic is in separate workflows + // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); + return processorManagerProvider.get().process(context.getBrAPIImports(), + List.of(experimentProcessorProvider.get()), + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); } } From 90f478becd85cd15db0d47a989ee399fb38d2256 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 10 Jun 2024 16:29:33 -0400 Subject: [PATCH 107/203] fix workflow navigator injection --- .../importer/model/imports/DomainImportService.java | 8 ++++---- .../experimentObservation/ExperimentImportService.java | 9 ++------- .../model/imports/germplasm/GermplasmImportService.java | 3 ++- .../imports/sample/SampleSubmissionImportService.java | 3 ++- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index d826033fb..13719e18d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -41,16 +41,16 @@ public abstract class DomainImportService implements BrAPIImportService { private final Provider processorManagerProvider; private final Workflow workflowNavigator; - @Inject + public DomainImportService(Provider experimentProcessorProvider, - Provider processorManagerProvider) + Provider processorManagerProvider, + Workflow workflowNavigator) { this.experimentProcessorProvider = experimentProcessorProvider; this.processorManagerProvider = processorManagerProvider; - this.workflowNavigator = getNavigator(); + this.workflowNavigator = workflowNavigator; } - protected abstract Workflow getNavigator(); @Override public String getMissingColumnMsg(String columnName) { return "Column heading does not match template or ontology"; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 60fe4bb3b..793506c73 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -38,19 +38,14 @@ public class ExperimentImportService extends DomainImportService { private final String IMPORT_TYPE_ID = "ExperimentImport"; - private final ExperimentWorkflowNavigator workflowNavigator; + // TODO: delete processor fields once WorkflowNavigator is used @Inject public ExperimentImportService(Provider experimentProcessorProvider, Provider processorManagerProvider, ExperimentWorkflowNavigator workflowNavigator) { - super(experimentProcessorProvider, processorManagerProvider); - this.workflowNavigator = workflowNavigator; - } - @Override - public Workflow getNavigator() { - return this.workflowNavigator; + super(experimentProcessorProvider, processorManagerProvider, workflowNavigator); } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index 3c3a65b4d..0caebe65e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -34,6 +34,7 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import java.util.ArrayList; import java.util.List; @Singleton @@ -60,7 +61,7 @@ public GermplasmImport getImportClass() { @Override public List getWorkflows() { - return null; + return new ArrayList<>(); } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java index 55848b0a1..eb7328ecf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java @@ -34,6 +34,7 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import java.util.ArrayList; import java.util.List; @Singleton @@ -62,7 +63,7 @@ public BrAPIImport getImportClass() { @Override public List getWorkflows() { - return null; + return new ArrayList<>(); } @Override From 62cf9ac438684991bb48358ca46eea1418a280eb Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:07:33 -0400 Subject: [PATCH 108/203] set the order field for workflows --- .../experiment/ExperimentWorkflowNavigator.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index b43b859f5..63bfcedbd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -29,12 +29,21 @@ public Optional process(ImportServiceContext context) { .findFirst(); } public List getWorkflows() { - return workflows.stream() + // Each workflow returns in the field workflow the metadata about the workflow that processed the import context. + // Loop over all workflows, processing a null context, to collect just the metadata + List workflowSummaryList = workflows.stream() .map(workflow->workflow.process(null)) .filter(Optional::isPresent) .map(Optional::get) .map(result->result.getWorkflow()) .collect(Collectors.toList()); + + // The order field for each workflow is set to the order in the list + for (int i = 0; i < workflowSummaryList.size(); i++) { + workflowSummaryList.get(i).setOrder(i); + } + + return workflowSummaryList; } public enum Workflow { From 57f5cfbfc10f937e1099fd5a017997a7102db78f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:43:08 -0400 Subject: [PATCH 109/203] fix npe --- .../brapps/importer/model/imports/DomainImportService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 13719e18d..af65af238 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -64,7 +64,7 @@ public List getWorkflows() { public ImportPreviewResponse process(ImportServiceContext context) throws Exception { - if (!context.getWorkflow().isEmpty()) { + if (context.getWorkflow() != null && !context.getWorkflow().isEmpty()) { log.info("Workflow: " + context.getWorkflow()); } From 6041deb4106e1e8d2d0668f15a46f1aa2de1a31a Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 10 Jun 2024 20:56:35 -0400 Subject: [PATCH 110/203] add workflow test util methods --- .../AppendOverwritePhenotypesWorkflow.java | 55 ------------------- .../importer/ExperimentFileImportTest.java | 10 +++- .../brapps/importer/ImportTestUtils.java | 51 +++++++++++++++++ 3 files changed, 60 insertions(+), 56 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java deleted file mode 100644 index 6ff074059..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; - -import lombok.Getter; -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; - -import javax.inject.Singleton; -import java.util.Optional; - -@Getter -@Singleton -public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow { - private final ExperimentWorkflowNavigator.Workflow workflow; - - public AppendOverwritePhenotypesWorkflow(){ - this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE; - } - - @Override - public Optional process(ImportServiceContext context) { - // Workflow processing the context - ImportWorkflow workflow = ImportWorkflow.builder() - .id(getWorkflow().getId()) - .name(getWorkflow().getName()) - .build(); - - // No-preview result - Optional result = Optional.of(ImportWorkflowResult.builder() - .workflow(workflow) - .importPreviewResponse(Optional.empty()) - .build()); - - // Skip this workflow unless appending or overwriting observation data - if (context != null && !this.workflow.isEqual(context.getWorkflow())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return result; - } - - // Start processing the import... - return result; - } - - @Override - public int getOrder() { - return 2; - } - -} diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 6bc203d64..37e2032ef 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1067,7 +1067,15 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + String workflowId = "append-dataset"; + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), + null, + commit, + client, + program, + mappingId, + workflowId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index 12b79ac15..c61348bff 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -97,6 +97,38 @@ public Flowable> uploadDataFile(File file, Map> uploadWorkflowDataFile(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) { + + MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); + + // Upload file + String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", program.getId(), mappingId); + Flowable> call = client.exchange( + POST(uploadUrl, requestBody) + .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + String importId = result.get("importId").getAsString(); + + // Process data + String url = String.format("/programs/%s/import/mappings/%s/workflows/%s/data/%s/%s", program.getId(), mappingId, workflowId, importId, commit ? "commit" : "preview"); + Flowable> processCall = client.exchange( + PUT(url, userData) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + return processCall; + + } + public HttpResponse getUploadedFile(String importId, RxHttpClient client, Program program, String mappingId) throws InterruptedException { Flowable> call = client.exchange( GET(String.format("/programs/%s/import/mappings/%s/data/%s?mapping=true", program.getId(), mappingId, importId)) @@ -170,6 +202,25 @@ public JsonObject uploadAndFetch(File file, Map userData, Boolea return result; } + public JsonObject uploadAndFetchWorkflow(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { + Flowable> call = uploadWorkflowDataFile(file, userData, commit, client, program, mappingId, workflowId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + return result; + } + public List createTraits(int numToCreate) { List traits = new ArrayList<>(); for (int i = 0; i < numToCreate; i++) { From 73da0a87300ac13275f87354b0e44126a9aefe14 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 11 Jun 2024 11:33:36 -0400 Subject: [PATCH 111/203] initialize expUnitContext --- .../AppendOverwritePhenotypesWorkflow.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index e0bcd6e9b..753144f9e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -13,6 +13,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.BrAPICommit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize.WorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.ImportTableProcess; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; @@ -47,8 +48,8 @@ public AppendOverwritePhenotypesWorkflow(Transaction transaction, public Optional process(ImportServiceContext context) { // Workflow processing the context ImportWorkflow workflow = ImportWorkflow.builder() - .urlFragment(getWorkflow().getUrlFragment()) - .displayName(getWorkflow().getDisplayName()) + .id(getWorkflow().getId()) + .name(getWorkflow().getName()) .build(); // No-preview result @@ -76,7 +77,10 @@ public Optional process(ImportServiceContext context) { .user(context.getUser()) .commit(context.isCommit()) .build(); - ExpUnitMiddlewareContext workflowContext = ExpUnitMiddlewareContext.builder().importContext(importContext).build(); + ExpUnitMiddlewareContext workflowContext = ExpUnitMiddlewareContext.builder() + .importContext(importContext) + .expUnitContext(new ExpUnitContext()) + .build(); // Process the workflow ExpUnitMiddlewareContext processedContext = this.middleware.process(workflowContext); From 01faa6b9aaa63a553b1c2e349e44a7a1072f27d3 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:31:23 -0400 Subject: [PATCH 112/203] fix dependency injection for pending obs unit --- .../model/imports/DomainImportService.java | 25 +++++++++++-------- ...rvationUnitReadWorkflowInitialization.java | 12 ++++++++- .../entity/PendingObservationUnit.java | 14 ++++++++--- .../service/ObservationUnitService.java | 2 ++ 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index ad711b744..071c9f0ef 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -29,6 +29,7 @@ import javax.inject.Provider; import javax.inject.Singleton; import java.util.List; +import java.util.Optional; @Singleton @Slf4j @@ -62,19 +63,23 @@ public List getWorkflows() { public ImportPreviewResponse process(ImportServiceContext context) throws Exception { - if (context.getWorkflow() != null && !context.getWorkflow().isEmpty()) { - log.info("Workflow: " + context.getWorkflow()); - } + Optional.ofNullable(context.getWorkflow()) + .filter(workflow -> !workflow.isEmpty()) + .ifPresent(workflow -> log.info("Workflow: " + workflow)); // TODO: return results from WorkflowNavigator once processing logic is in separate workflows // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); - return processorManagerProvider.get().process(context.getBrAPIImports(), - List.of(experimentProcessorProvider.get()), - context.getData(), - context.getProgram(), - context.getUpload(), - context.getUser(), - context.isCommit()); + if ("append-dataset".equals(context.getWorkflow())) { + return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); + } else { + return processorManagerProvider.get().process(context.getBrAPIImports(), + List.of(experimentProcessorProvider.get()), + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); + } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java index cd8d20dea..6bf97b8a9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java @@ -1,9 +1,13 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +import io.micronaut.context.ApplicationContext; import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; public class BrAPIObservationUnitReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { /** @@ -24,6 +28,12 @@ public BrAPIObservationUnitReadWorkflowInitialization(ExpUnitMiddlewareContext c */ @Override public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - return new PendingObservationUnit(context); + try (ApplicationContext appContext = ApplicationContext.run()) { + BrAPIObservationUnitDAO observationUnitDAO = appContext.getBean(BrAPIObservationUnitDAO.class); + ObservationUnitService observationUnitService = appContext.getBean(ObservationUnitService.class); + ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); + + return new PendingObservationUnit(context, observationUnitDAO, observationUnitService, experimentUtilities); + } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java index a10681e7c..41f404629 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; +import io.micronaut.context.annotation.Prototype; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservationUnit; @@ -21,18 +22,23 @@ import java.util.*; import java.util.stream.Collectors; +@Prototype public class PendingObservationUnit implements ExperimentImportEntity { ExpUnitContext cache; ImportContext importContext; - @Inject BrAPIObservationUnitDAO observationUnitDAO; - @Inject ObservationUnitService observationUnitService; - @Inject ExperimentUtilities experimentUtilities; - public PendingObservationUnit(ExpUnitMiddlewareContext context) { + @Inject + public PendingObservationUnit(ExpUnitMiddlewareContext context, + BrAPIObservationUnitDAO observationUnitDAO, + ObservationUnitService observationUnitService, + ExperimentUtilities experimentUtilities) { this.cache = context.getExpUnitContext(); this.importContext = context.getImportContext(); + this.observationUnitDAO = observationUnitDAO; + this.observationUnitService = observationUnitService; + this.experimentUtilities = experimentUtilities; } /** * Create new objects generated by the workflow in the BrAPI service. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index 708b2b481..489c36f0a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -27,11 +27,13 @@ import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; +import javax.inject.Singleton; import java.util.*; import java.util.stream.Collectors; import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; +@Singleton public class ObservationUnitService { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; @Property(name = "brapi.server.reference-source") From 5dab53d2de9a59d05252e34aa5559e8f29cbb60b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:43:12 -0400 Subject: [PATCH 113/203] fix injection for pending OU creation --- .../action/create/BrAPIObservationUnitCreation.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationUnitCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationUnitCreation.java index 87b5105d4..21e492415 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationUnitCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationUnitCreation.java @@ -1,9 +1,13 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; +import io.micronaut.context.ApplicationContext; import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; public class BrAPIObservationUnitCreation extends BrAPICreation { /** @@ -23,6 +27,12 @@ public BrAPIObservationUnitCreation(ExpUnitMiddlewareContext context) { */ @Override public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - return new PendingObservationUnit(context); + try (ApplicationContext appContext = ApplicationContext.run()) { + BrAPIObservationUnitDAO observationUnitDAO = appContext.getBean(BrAPIObservationUnitDAO.class); + ObservationUnitService observationUnitService = appContext.getBean(ObservationUnitService.class); + ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); + + return new PendingObservationUnit(context, observationUnitDAO, observationUnitService, experimentUtilities); + } } } From 5dda5a564380a45533988756b75d362986d82ce5 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 11 Jun 2024 17:16:07 -0400 Subject: [PATCH 114/203] fix injection pending trial entity --- .../action/create/BrAPITrialCreation.java | 12 +++++++++++- .../appendoverwrite/action/read/BrAPITrialRead.java | 12 +++++++++++- .../read/BrAPITrialReadWorkflowInitialization.java | 12 +++++++++++- .../action/update/BrAPITrialUpdate.java | 12 +++++++++++- .../appendoverwrite/entity/PendingTrial.java | 12 ++++++++---- 5 files changed, 52 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPITrialCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPITrialCreation.java index 4d0f7aa2c..71fa1ba14 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPITrialCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPITrialCreation.java @@ -1,10 +1,14 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; +import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingTrial; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; @Slf4j public class BrAPITrialCreation extends BrAPICreation { @@ -15,6 +19,12 @@ public BrAPITrialCreation(ExpUnitMiddlewareContext context) { @Override public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - return new PendingTrial(context); + try (ApplicationContext appContext = ApplicationContext.run()) { + TrialService trialService = appContext.getBean(TrialService.class); + BrAPITrialDAO brAPITrialDAO = appContext.getBean(BrAPITrialDAO.class); + ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); + + return new PendingTrial(context, trialService, brAPITrialDAO, experimentUtilities); + } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialRead.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialRead.java index 525f4ec9c..47e484e48 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialRead.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialRead.java @@ -1,10 +1,14 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingTrial; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; @Slf4j public class BrAPITrialRead extends BrAPIRead { @@ -15,6 +19,12 @@ public BrAPITrialRead(ExpUnitMiddlewareContext context) { @Override public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - return new PendingTrial(context); + try (ApplicationContext appContext = ApplicationContext.run()) { + TrialService trialService = appContext.getBean(TrialService.class); + BrAPITrialDAO brAPITrialDAO = appContext.getBean(BrAPITrialDAO.class); + ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); + + return new PendingTrial(context, trialService, brAPITrialDAO, experimentUtilities); + } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialReadWorkflowInitialization.java index e3aeb2da4..e7a49b418 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialReadWorkflowInitialization.java @@ -1,11 +1,15 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.BrAPIReadWorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingTrial; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; @Slf4j public class BrAPITrialReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { @@ -16,6 +20,12 @@ public BrAPITrialReadWorkflowInitialization(ExpUnitMiddlewareContext context) { @Override public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - return new PendingTrial(context); + try (ApplicationContext appContext = ApplicationContext.run()) { + TrialService trialService = appContext.getBean(TrialService.class); + BrAPITrialDAO brAPITrialDAO = appContext.getBean(BrAPITrialDAO.class); + ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); + + return new PendingTrial(context, trialService, brAPITrialDAO, experimentUtilities); + } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPITrialUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPITrialUpdate.java index 7303eef32..dc9bba419 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPITrialUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPITrialUpdate.java @@ -1,10 +1,14 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update; +import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingTrial; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; @Slf4j public class BrAPITrialUpdate extends BrAPIUpdate { @@ -15,6 +19,12 @@ public BrAPITrialUpdate(ExpUnitMiddlewareContext context) { @Override public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - return new PendingTrial(context); + try (ApplicationContext appContext = ApplicationContext.run()) { + TrialService trialService = appContext.getBean(TrialService.class); + BrAPITrialDAO brAPITrialDAO = appContext.getBean(BrAPITrialDAO.class); + ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); + + return new PendingTrial(context, trialService, brAPITrialDAO, experimentUtilities); + } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java index 6a9a5e809..913bb6f11 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java @@ -19,16 +19,20 @@ public class PendingTrial implements ExperimentImportEntity { ExpUnitContext cache; ImportContext importContext; - @Inject TrialService trialService; - @Inject BrAPITrialDAO brapiTrialDAO; - @Inject ExperimentUtilities experimentUtilities; - public PendingTrial(ExpUnitMiddlewareContext context) { + @Inject + public PendingTrial(ExpUnitMiddlewareContext context, + TrialService trialService, + BrAPITrialDAO brAPITrialDAO, + ExperimentUtilities experimentUtilities) { this.cache = context.getExpUnitContext(); this.importContext = context.getImportContext(); + this.trialService = trialService; + this.brapiTrialDAO = brAPITrialDAO; + this.experimentUtilities = experimentUtilities; } /** From 49e89bbe1644a32d8a4cb1af1272370b59b89cb9 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 11 Jun 2024 17:29:42 -0400 Subject: [PATCH 115/203] fix injection pending study --- .../action/create/BrAPIStudyCreation.java | 12 +++++++++++- .../read/BrAPIStudyReadWorkflowInitialization.java | 12 +++++++++++- .../appendoverwrite/entity/PendingStudy.java | 12 ++++++++---- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIStudyCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIStudyCreation.java index 06eaae555..1020c0402 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIStudyCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIStudyCreation.java @@ -1,10 +1,14 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; +import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingStudy; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; @Slf4j public class BrAPIStudyCreation extends BrAPICreation { @@ -21,6 +25,12 @@ public BrAPIStudyCreation(ExpUnitMiddlewareContext context) { */ @Override public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - return new PendingStudy(context); + try (ApplicationContext appContext = ApplicationContext.run()) { + StudyService studyService = appContext.getBean(StudyService.class); + BrAPIStudyDAO brAPIStudyDAO = appContext.getBean(BrAPIStudyDAO.class); + ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); + + return new PendingStudy(context, studyService, brAPIStudyDAO, experimentUtilities); + } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java index 1c06cc3b8..3d75fc285 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java @@ -1,9 +1,13 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +import io.micronaut.context.ApplicationContext; import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingStudy; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; public class BrAPIStudyReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { @@ -19,6 +23,12 @@ public BrAPIStudyReadWorkflowInitialization(ExpUnitMiddlewareContext context) { */ @Override public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - return new PendingStudy(context); + try (ApplicationContext appContext = ApplicationContext.run()) { + StudyService studyService = appContext.getBean(StudyService.class); + BrAPIStudyDAO brAPIStudyDAO = appContext.getBean(BrAPIStudyDAO.class); + ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); + + return new PendingStudy(context, studyService, brAPIStudyDAO, experimentUtilities); + } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java index faaf18615..dbb8a53d4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java @@ -25,16 +25,20 @@ public class PendingStudy implements ExperimentImportEntity{ ExpUnitContext cache; ImportContext importContext; - @Inject StudyService studyService; - @Inject BrAPIStudyDAO brAPIStudyDAO; - @Inject ExperimentUtilities experimentUtilities; - public PendingStudy(ExpUnitMiddlewareContext context) { + @Inject + public PendingStudy(ExpUnitMiddlewareContext context, + StudyService studyService, + BrAPIStudyDAO brAPIStudyDAO, + ExperimentUtilities experimentUtilities) { this.cache = context.getExpUnitContext(); this.importContext = context.getImportContext(); + this.studyService = studyService; + this.brAPIStudyDAO = brAPIStudyDAO; + this.experimentUtilities = experimentUtilities; } /** * Create new objects generated by the workflow in the BrAPI service. From 535b001d882e1011c1d0dad090643be3648ed971 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 11 Jun 2024 17:44:15 -0400 Subject: [PATCH 116/203] fix injection pending location --- .../action/create/LocationCreation.java | 12 +++++++++++- .../read/LocationReadWorkflowInitialization.java | 12 +++++++++++- .../appendoverwrite/entity/PendingLocation.java | 12 ++++++++---- .../experiment/service/DatasetService.java | 2 ++ .../experiment/service/GermplasmService.java | 2 ++ .../service/ObservationVariableService.java | 2 ++ 6 files changed, 36 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/LocationCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/LocationCreation.java index a3def6c0c..9ae5167ee 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/LocationCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/LocationCreation.java @@ -1,9 +1,13 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; +import io.micronaut.context.ApplicationContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingLocation; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.ProgramLocationService; public class LocationCreation extends BrAPICreation{ /** @@ -23,6 +27,12 @@ public LocationCreation(ExpUnitMiddlewareContext context) { */ @Override public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - return new PendingLocation(context); + try (ApplicationContext appContext = ApplicationContext.run()) { + ProgramLocationService programLocationService = appContext.getBean(ProgramLocationService.class); + LocationService locationService = appContext.getBean(LocationService.class); + ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); + + return new PendingLocation(context, programLocationService, locationService, experimentUtilities); + } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java index b072e65b5..e0196d8d8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java @@ -1,10 +1,14 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingLocation; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.ProgramLocationService; @Slf4j public class LocationReadWorkflowInitialization extends BrAPIReadWorkflowInitialization{ @@ -26,6 +30,12 @@ public LocationReadWorkflowInitialization(ExpUnitMiddlewareContext context) { */ @Override public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - return new PendingLocation(context); + try (ApplicationContext appContext = ApplicationContext.run()) { + ProgramLocationService programLocationService = appContext.getBean(ProgramLocationService.class); + LocationService locationService = appContext.getBean(LocationService.class); + ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); + + return new PendingLocation(context, programLocationService, locationService, experimentUtilities); + } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java index d934c9ec6..03607bd1b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java @@ -26,15 +26,19 @@ public class PendingLocation implements ExperimentImportEntity { ExpUnitContext cache; ImportContext importContext; - @Inject ProgramLocationService programLocationService; - @Inject LocationService locationService; - @Inject ExperimentUtilities experimentUtilities; - public PendingLocation(ExpUnitMiddlewareContext context) { + @Inject + public PendingLocation(ExpUnitMiddlewareContext context, + ProgramLocationService programLocationService, + LocationService locationService, + ExperimentUtilities experimentUtilities) { this.cache = context.getExpUnitContext(); this.importContext = context.getImportContext(); + this.programLocationService = programLocationService; + this.locationService = locationService; + this.experimentUtilities = experimentUtilities; } /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index 9810f8c62..b95a6d399 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -25,9 +25,11 @@ import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; +import javax.inject.Singleton; import java.util.*; import java.util.stream.Collectors; +@Singleton public class DatasetService { private final BrAPIListDAO brAPIListDAO; @Property(name = "brapi.server.reference-source") diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java index 88d085e41..a6ffd6610 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java @@ -15,9 +15,11 @@ import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; +import javax.inject.Singleton; import java.util.*; import java.util.stream.Collectors; +@Singleton public class GermplasmService { private final BrAPIGermplasmDAO germplasmDAO; @Property(name = "brapi.server.reference-source") diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java index fdf692df9..f0e55b57c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java @@ -28,11 +28,13 @@ import org.breedinginsight.dao.db.tables.pojos.TraitEntity; import javax.inject.Inject; +import javax.inject.Singleton; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.*; import java.util.stream.Collectors; +@Singleton public class ObservationVariableService { private final OntologyService ontologyService; From d91c266af8c2686d9c333e3b451988fba29bf199 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 11 Jun 2024 17:55:19 -0400 Subject: [PATCH 117/203] fix injection pendign observation --- .../action/create/BrAPIObservationCreation.java | 12 +++++++++++- .../action/update/BrAPIObservationUpdate.java | 12 +++++++++++- .../appendoverwrite/entity/PendingObservation.java | 14 ++++++++------ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationCreation.java index 74c430fde..e204068b6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationCreation.java @@ -1,10 +1,14 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; +import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingObservation; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.services.OntologyService; @Slf4j public class BrAPIObservationCreation extends BrAPICreation { @@ -15,6 +19,12 @@ public BrAPIObservationCreation(ExpUnitMiddlewareContext context) { @Override public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - return new PendingObservation(context); + try (ApplicationContext appContext = ApplicationContext.run()) { + BrAPIObservationDAO brAPIObservationDAO = appContext.getBean(BrAPIObservationDAO.class); + OntologyService ontologyService = appContext.getBean(OntologyService.class); + ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); + + return new PendingObservation(context, brAPIObservationDAO, ontologyService, experimentUtilities); + } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIObservationUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIObservationUpdate.java index aa2649320..5cf4cec56 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIObservationUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIObservationUpdate.java @@ -1,10 +1,14 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update; +import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingObservation; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.services.OntologyService; @Slf4j public class BrAPIObservationUpdate extends BrAPIUpdate { @@ -15,6 +19,12 @@ public BrAPIObservationUpdate(ExpUnitMiddlewareContext context) { @Override public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - return new PendingObservation(context); + try (ApplicationContext appContext = ApplicationContext.run()) { + BrAPIObservationDAO brAPIObservationDAO = appContext.getBean(BrAPIObservationDAO.class); + OntologyService ontologyService = appContext.getBean(OntologyService.class); + ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); + + return new PendingObservation(context, brAPIObservationDAO, ontologyService, experimentUtilities); + } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java index bd0dae932..1d55c43d2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java @@ -23,18 +23,20 @@ public class PendingObservation implements ExperimentImportEntity { ExpUnitContext cache; ImportContext importContext; - @Inject - ObservationService observationService; - @Inject BrAPIObservationDAO brAPIObservationDAO; - @Inject OntologyService ontologyService; - @Inject ExperimentUtilities experimentUtilities; - public PendingObservation(ExpUnitMiddlewareContext context) { + @Inject + public PendingObservation(ExpUnitMiddlewareContext context, + BrAPIObservationDAO brAPIObservationDAO, + OntologyService ontologyService, + ExperimentUtilities experimentUtilities) { this.cache = context.getExpUnitContext(); this.importContext = context.getImportContext(); + this.brAPIObservationDAO = brAPIObservationDAO; + this.ontologyService = ontologyService; + this.experimentUtilities = experimentUtilities; } From 06ba0388561111f8a8ac07ef08878b4d9f1128cc Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 11 Jun 2024 18:04:32 -0400 Subject: [PATCH 118/203] fix injection pending germplasm --- .../read/BrAPIGermplasmReadWorkflowInitialization.java | 10 +++++++++- .../appendoverwrite/entity/PendingGermplasm.java | 9 ++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java index 2c45f1b68..b60c8735d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java @@ -1,9 +1,12 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +import io.micronaut.context.ApplicationContext; import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingGermplasm; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.GermplasmService; public class BrAPIGermplasmReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { /** @@ -24,6 +27,11 @@ public BrAPIGermplasmReadWorkflowInitialization(ExpUnitMiddlewareContext context */ @Override public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - return new PendingGermplasm(context); + try (ApplicationContext appContext = ApplicationContext.run()) { + GermplasmService germplasmService = appContext.getBean(GermplasmService.class); + ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); + + return new PendingGermplasm(context, germplasmService, experimentUtilities); + } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java index 28e28d52e..624f48b57 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java @@ -20,14 +20,17 @@ public class PendingGermplasm implements ExperimentImportEntity { ExpUnitContext cache; ImportContext importContext; - @Inject GermplasmService germplasmService; - @Inject ExperimentUtilities experimentUtilities; - public PendingGermplasm(ExpUnitMiddlewareContext context) { + @Inject + public PendingGermplasm(ExpUnitMiddlewareContext context, + GermplasmService germplasmService, + ExperimentUtilities experimentUtilities) { this.cache = context.getExpUnitContext(); this.importContext = context.getImportContext(); + this.germplasmService = germplasmService; + this.experimentUtilities = experimentUtilities; } /** * Create new objects generated by the workflow in the BrAPI service. From b106b2d6ef95d9760add91fd69fffd508810173d Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 11 Jun 2024 18:11:57 -0400 Subject: [PATCH 119/203] fix injection pending dataset --- .../action/create/BrAPIDatasetCreation.java | 12 +++++++++++- .../read/BrAPIDatasetReadWorkflowInitialization.java | 12 +++++++++++- .../appendoverwrite/entity/PendingDataset.java | 12 ++++++++---- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java index 479e8b4e9..5b2026a0d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java @@ -1,10 +1,14 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; +import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingDataset; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; @Slf4j public class BrAPIDatasetCreation extends BrAPICreation { @@ -25,6 +29,12 @@ public BrAPIDatasetCreation(ExpUnitMiddlewareContext context) { */ @Override public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - return new PendingDataset(context); + try (ApplicationContext appContext = ApplicationContext.run()) { + BrAPIListDAO brAPIListDAO = appContext.getBean(BrAPIListDAO.class); + DatasetService datasetService = appContext.getBean(DatasetService.class); + ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); + + return new PendingDataset(context, brAPIListDAO, datasetService, experimentUtilities); + } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java index 8c325bb72..d6d3f9f69 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java @@ -1,9 +1,13 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +import io.micronaut.context.ApplicationContext; import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingDataset; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; public class BrAPIDatasetReadWorkflowInitialization extends BrAPIReadWorkflowInitialization{ /** @@ -24,6 +28,12 @@ public BrAPIDatasetReadWorkflowInitialization(ExpUnitMiddlewareContext context) */ @Override public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - return new PendingDataset(context); + try (ApplicationContext appContext = ApplicationContext.run()) { + BrAPIListDAO brAPIListDAO = appContext.getBean(BrAPIListDAO.class); + DatasetService datasetService = appContext.getBean(DatasetService.class); + ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); + + return new PendingDataset(context, brAPIListDAO, datasetService, experimentUtilities); + } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java index b9b9b7abf..4c7770b90 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java @@ -26,15 +26,19 @@ public class PendingDataset implements ExperimentImportEntity { ExpUnitContext cache; ImportContext importContext; - @Inject BrAPIListDAO brAPIListDAO; - @Inject DatasetService datasetService; - @Inject ExperimentUtilities experimentUtilities; - public PendingDataset(ExpUnitMiddlewareContext context) { + @Inject + public PendingDataset(ExpUnitMiddlewareContext context, + BrAPIListDAO brAPIListDAO, + DatasetService datasetService, + ExperimentUtilities experimentUtilities) { this.cache = context.getExpUnitContext(); this.importContext = context.getImportContext(); + this.brAPIListDAO = brAPIListDAO; + this.datasetService = datasetService; + this.experimentUtilities = experimentUtilities; } /** * Create new objects generated by the workflow in the BrAPI service. From 45d72d01fe6c88f4d63ce787f4227b14674be87d Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 11 Jun 2024 21:51:59 -0400 Subject: [PATCH 120/203] fix workflwo test util and pending trial NPE --- .../appendoverwrite/entity/PendingTrial.java | 8 ++++++-- .../importer/ExperimentFileImportTest.java | 13 ++++--------- .../brapps/importer/ImportTestUtils.java | 17 ++++++++++++++++- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java index 913bb6f11..7c8d3ad08 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java @@ -2,6 +2,7 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -56,8 +57,11 @@ public List brapiPost(List members) throws ApiException @Override public List brapiRead() throws ApiException { // Get the dbIds of the trials belonging to the required exp units - Set trialDbIds = cache.getObservationUnitByNameNoScope().values().stream() - .map(pendingUnit -> trialService.getTrialDbIdBelongingToPendingUnit(pendingUnit, importContext.getProgram())).collect(Collectors.toSet()); + Set trialDbIds = Optional.ofNullable(cache.getObservationUnitByNameNoScope()).map(Map::values) + .orElse(Collections.emptySet()) + .stream() + .map(pendingUnit -> trialService.getTrialDbIdBelongingToPendingUnit(pendingUnit, importContext.getProgram())) + .collect(Collectors.toSet()); // Get the BrAPI trials belonging to required exp units return trialService.fetchBrapiTrialsByDbId(trialDbIds, importContext.getProgram()); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 37e2032ef..5f037c4af 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1038,7 +1038,8 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + String workflowId = "append-dataset"; + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, workflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1067,14 +1068,8 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - String workflowId = "append-dataset"; - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), - null, - commit, - client, - program, - mappingId, - workflowId); + + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, workflowId); //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index c61348bff..7acfed069 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -129,6 +129,21 @@ public Flowable> uploadWorkflowDataFile(File file, } + public HttpResponse getUploadedFileWorkflow(String importId, RxHttpClient client, Program program, String mappingId, String workflowId) throws InterruptedException { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/import/mappings/%s/workflows/%s/data/%s?mapping=true", program.getId(), mappingId, workflowId, importId)) + .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + + if (response.getStatus().equals(HttpStatus.ACCEPTED)) { + Thread.sleep(1000); + return getUploadedFile(importId, client, program, mappingId); + } else { + return response; + } + } public HttpResponse getUploadedFile(String importId, RxHttpClient client, Program program, String mappingId) throws InterruptedException { Flowable> call = client.exchange( GET(String.format("/programs/%s/import/mappings/%s/data/%s?mapping=true", program.getId(), mappingId, importId)) @@ -215,7 +230,7 @@ public JsonObject uploadAndFetchWorkflow(File file, String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId, client, program, mappingId); + HttpResponse upload = getUploadedFileWorkflow(importId, client, program, mappingId, workflowId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); return result; From a44896b81a62b8751c4f7c8ab1e757ec49586bb0 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 12 Jun 2024 10:41:46 -0400 Subject: [PATCH 121/203] fix workflow test util --- .../brapps/importer/ImportTestUtils.java | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index 7acfed069..2229b609b 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -128,22 +128,6 @@ public Flowable> uploadWorkflowDataFile(File file, return processCall; } - - public HttpResponse getUploadedFileWorkflow(String importId, RxHttpClient client, Program program, String mappingId, String workflowId) throws InterruptedException { - Flowable> call = client.exchange( - GET(String.format("/programs/%s/import/mappings/%s/workflows/%s/data/%s?mapping=true", program.getId(), mappingId, workflowId, importId)) - .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - HttpResponse response = call.blockingFirst(); - - if (response.getStatus().equals(HttpStatus.ACCEPTED)) { - Thread.sleep(1000); - return getUploadedFile(importId, client, program, mappingId); - } else { - return response; - } - } public HttpResponse getUploadedFile(String importId, RxHttpClient client, Program program, String mappingId) throws InterruptedException { Flowable> call = client.exchange( GET(String.format("/programs/%s/import/mappings/%s/data/%s?mapping=true", program.getId(), mappingId, importId)) @@ -230,7 +214,7 @@ public JsonObject uploadAndFetchWorkflow(File file, String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFileWorkflow(importId, client, program, mappingId, workflowId); + HttpResponse upload = getUploadedFile(importId, client, program, mappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); return result; From 644ca0c8e0b7a51080f7d647d90de54160ec481c Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 12 Jun 2024 14:54:25 -0400 Subject: [PATCH 122/203] Added commit step --- .../ExperimentImportService.java | 15 + .../importer/model/workflow/Workflow.java | 2 +- .../experiment/ExperimentUtilities.java | 215 +++++++ .../experiment/create/model/PendingData.java | 5 + .../workflow/CreateNewExperimentWorkflow.java | 16 +- .../CreatePendingImportPopulator.java | 80 --- .../steps/CommitPendingImportObjectsStep.java | 305 ++++++++- ...ulateExistingPendingImportObjectsStep.java | 8 +- .../PopulateNewPendingImportObjectsStep.java | 308 +++++---- .../create/workflow/steps/ProcessStep.java | 597 ------------------ .../ValidatePendingImportObjectsStep.java | 464 ++++++++++++++ .../services/ExperimentSeasonService.java | 5 +- .../services/ExperimentValidateService.java | 2 + .../PendingImportObjectPopulator.java | 45 -- 14 files changed, 1198 insertions(+), 869 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index c6f68b251..fededeea1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -23,6 +23,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; @@ -75,9 +76,23 @@ public ImportPreviewResponse process(ImportServiceContext context) if (context.getWorkflow() != null) { log.info("Workflow: " + context.getWorkflow().getName()); + + // TODO: change when workflows selection is ready + if (context.getWorkflow().getName().equals("CreateNewExperimentWorkflow")) { + ImportContext importContext = ImportContext.builder() + .importRows(context.getBrAPIImports()) + .user(context.getUser()) + .data(context.getData()) + .commit(context.isCommit()) + .upload(context.getUpload()) + .build(); + + return context.getWorkflow().process(importContext); + } } // TODO: change to calling workflow process instead of processor manager + // other workflows besides create will pass through to old flow response = processorManagerProvider.get().process(context.getBrAPIImports(), processors, context.getData(), diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java index 8b2165933..5d8cbe66e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -27,7 +27,7 @@ public interface Workflow { * @param context the import context containing the necessary data for processing * @return the processed data */ - ImportPreviewResponse process(ImportContext context); + ImportPreviewResponse process(ImportContext context) throws Exception; /** * Retrieves the name of the Workflow for logging display purposes. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 8092a01e9..f3b57fb03 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -16,16 +16,36 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment; +import com.google.gson.JsonObject; +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Scale; +import org.breedinginsight.model.Trait; +import tech.tablesaw.columns.Column; +import java.math.BigDecimal; +import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.*; import java.util.stream.Collectors; +@Slf4j public class ExperimentUtilities { public static final CharSequence COMMA_DELIMITER = ","; @@ -34,6 +54,7 @@ public class ExperimentUtilities { public static final String MIDNIGHT = "T00:00:00-00:00"; public static final String MULTIPLE_EXP_TITLES = "File contains more than one Experiment Title"; public static final String PREEXISTING_EXPERIMENT_TITLE = "Experiment Title already exists"; + public static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs"; @@ -59,4 +80,198 @@ public static String getObservationHash(String observationUnitName, String varia DigestUtils.sha256Hex(StringUtils.defaultString(studyName)); return DigestUtils.sha256Hex(concat); } + + /* + * this finds the YEAR from the season list on the BrAPIStudy and then + * will add the year to the additionalInfo-field of the BrAPIStudy + * */ + public static void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study) { + JsonObject additionalInfo = study.getAdditionalInfo(); + + //if it is already there, don't add it. + if(additionalInfo==null || additionalInfo.get(BrAPIAdditionalInfoFields.ENV_YEAR)==null) { + String year = study.getSeasons().get(0); + addYearToStudyAdditionalInfo(program, study, year); + } + } + + /* + * this will add the given year to the additionalInfo field of the BrAPIStudy (if it does not already exist) + * */ + public static void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study, String year) { + JsonObject additionalInfo = study.getAdditionalInfo(); + if (additionalInfo==null){ + additionalInfo = new JsonObject(); + study.setAdditionalInfo(additionalInfo); + } + if( additionalInfo.get(BrAPIAdditionalInfoFields.ENV_YEAR)==null) { + additionalInfo.addProperty(BrAPIAdditionalInfoFields.ENV_YEAR, year); + } + } + + public static String createObservationUnitKey(ExperimentObservation importRow) { + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); + } + + public static String createObservationUnitKey(String studyName, String obsUnitName) { + return studyName + obsUnitName; + } + + public static String getImportObservationHash(ExperimentObservation importRow, String variableName) { + return getObservationHash(createObservationUnitKey(importRow), variableName, importRow.getEnv()); + } + + public static String getVariableNameFromColumn(Column column) { + // TODO: timestamp stripping? + return column.name(); + } + + // TODO: common validation stuff, could probably be moved somewhere more specific to validation + public static void addRowError(String field, String errorMessage, ValidationErrors validationErrors, int rowNum) { + ValidationError ve = new ValidationError(field, errorMessage, HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(rowNum + 2, ve); // +2 instead of +1 to account for the column header row. + } + + // TODO: will have different pending data objects between workflows so not totally reusable as-is + // could probably just pass in actual underlying maps + public static boolean isObservationMatched(ProcessedPhenotypeData phenotypeData, + PendingData pendingData, + String observationHash, + String value, + Column phenoCol, + Integer rowNum) { + Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); + + if (timeStampColByPheno.isEmpty() || !timeStampColByPheno.containsKey(phenoCol.name())) { + return isValueMatched(pendingData, observationHash, value); + } else { + String importObsTimestamp = timeStampColByPheno.get(phenoCol.name()).getString(rowNum); + return isTimestampMatched(pendingData, observationHash, importObsTimestamp) && isValueMatched(pendingData, observationHash, value); + } + } + + // TODO: will have different pending data objects between workflows so not totally reusable as-is + // could probably just pass in actual underlying maps + public static boolean isValueMatched(PendingData pendingData, String observationHash, String value) { + Map existingObsByObsHash = pendingData.getExistingObsByObsHash(); + + if (!existingObsByObsHash.containsKey(observationHash) || existingObsByObsHash.get(observationHash).getValue() == null) { + return value == null; + } + return existingObsByObsHash.get(observationHash).getValue().equals(value); + } + + // TODO: will have different pending data objects between workflows so not totally reusable as-is + // could probably just pass in actual underlying maps + public static boolean isTimestampMatched(PendingData pendingData, String observationHash, String timeStamp) { + OffsetDateTime priorStamp = null; + Map existingObsByObsHash = pendingData.getExistingObsByObsHash(); + + if(existingObsByObsHash.get(observationHash)!=null){ + priorStamp = existingObsByObsHash.get(observationHash).getObservationTimeStamp(); + } + if (priorStamp == null) { + return timeStamp == null; + } + boolean isMatched = false; + try { + isMatched = priorStamp.isEqual(OffsetDateTime.parse(timeStamp)); + } catch(DateTimeParseException e){ + //if timestamp is invalid DateTime not equal to validated priorStamp + log.error(e.getMessage(), e); + } + return isMatched; + } + + public static void validateObservationValue(Trait variable, String value, + String columnHeader, ValidationErrors validationErrors, int row) { + if (StringUtils.isBlank(value)) { + log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s\n\trow: %d", variable.getObservationVariableName(), row)); + return; + } + + if (isNAObservation(value)) { + log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s\n\trow: %d", variable.getObservationVariableName(), row)); + return; + } + + switch (variable.getScale().getDataType()) { + case NUMERICAL: + Optional number = validNumericValue(value); + if (number.isEmpty()) { + addRowError(columnHeader, "Non-numeric text detected detected", validationErrors, row); + } else if (!validNumericRange(number.get(), variable.getScale())) { + addRowError(columnHeader, "Value outside of min/max range detected", validationErrors, row); + } + break; + case DATE: + if (!validDateValue(value)) { + addRowError(columnHeader, "Incorrect date format detected. Expected YYYY-MM-DD", validationErrors, row); + } + break; + case ORDINAL: + if (!validCategory(variable.getScale().getCategories(), value)) { + addRowError(columnHeader, "Undefined ordinal category detected", validationErrors, row); + } + break; + case NOMINAL: + if (!validCategory(variable.getScale().getCategories(), value)) { + addRowError(columnHeader, "Undefined nominal category detected", validationErrors, row); + } + break; + default: + break; + } + + } + + public static Optional validNumericValue(String value) { + BigDecimal number; + try { + number = new BigDecimal(value); + } catch (NumberFormatException e) { + return Optional.empty(); + } + return Optional.of(number); + } + + public static boolean validNumericRange(BigDecimal value, Scale validValues) { + // account for empty min or max in valid determination + return (validValues.getValidValueMin() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMin())) >= 0) && + (validValues.getValidValueMax() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMax())) <= 0); + } + + public static boolean validDateValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } + + public static boolean validCategory(List categories, String value) { + Set categoryValues = categories.stream() + .map(category -> category.getValue().toLowerCase()) + .collect(Collectors.toSet()); + return categoryValues.contains(value.toLowerCase()); + } + + public static boolean isNAObservation(String value){ + return value.equalsIgnoreCase("NA"); + } + + public static void validateTimeStampValue(String value, + String columnHeader, ValidationErrors validationErrors, int row) { + if (StringUtils.isBlank(value)) { + log.debug(String.format("skipping validation of observation timestamp because there is no value.\n\tvariable: %s\n\trow: %d", columnHeader, row)); + return; + } + if (!validDateValue(value) && !validDateTimeValue(value)) { + addRowError(columnHeader, "Incorrect datetime format detected. Expected YYYY-MM-DD or YYYY-MM-DDThh:mm:ss+hh:mm", validationErrors, row); + } + + } + } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java index 96c54712b..70ba1b62a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -26,6 +26,7 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.model.ProgramLocation; +import java.util.HashMap; import java.util.Map; @Getter @@ -35,6 +36,7 @@ @AllArgsConstructor @NoArgsConstructor public class PendingData { + //NOTE: populated in populate existing and new steps depending on import private Map> observationUnitByNameNoScope; private Map> trialByNameNoScope; private Map> studyByNameNoScope; @@ -43,4 +45,7 @@ public class PendingData { private Map> existingGermplasmByGID; // TODO: see if we can change this to match PendingImport<> private Map existingObsByObsHash; + + // NOTE: populated in populate new step + private Map> observationByHash; } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index e59226b81..e32a5c29c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -21,7 +21,9 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; +import lombok.val; import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; @@ -37,7 +39,9 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.CommitPendingImportObjectsStep; import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.PopulateExistingPendingImportObjectsStep; import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.PopulateNewPendingImportObjectsStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.ValidatePendingImportObjectsStep; import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentPhenotypeService; +import org.breedinginsight.services.exceptions.ValidatorException; import javax.inject.Inject; import javax.inject.Named; @@ -58,6 +62,7 @@ public class CreateNewExperimentWorkflow implements Workflow { private final PopulateExistingPendingImportObjectsStep populateExistingPendingImportObjectsStep; private final PopulateNewPendingImportObjectsStep populateNewPendingImportObjectsStep; private final CommitPendingImportObjectsStep commitPendingImportObjectsStep; + private final ValidatePendingImportObjectsStep validatePendingImportObjectsStep; private final ImportStatusService statusService; private final ExperimentPhenotypeService experimentPhenotypeService; @@ -65,17 +70,19 @@ public class CreateNewExperimentWorkflow implements Workflow { public CreateNewExperimentWorkflow(PopulateExistingPendingImportObjectsStep populateExistingPendingImportObjectsStep, PopulateNewPendingImportObjectsStep populateNewPendingImportObjectsStep, CommitPendingImportObjectsStep commitPendingImportObjectsStep, + ValidatePendingImportObjectsStep validatePendingImportObjectsStep, ImportStatusService statusService, ExperimentPhenotypeService experimentPhenotypeService) { this.populateExistingPendingImportObjectsStep = populateExistingPendingImportObjectsStep; this.populateNewPendingImportObjectsStep = populateNewPendingImportObjectsStep; this.commitPendingImportObjectsStep = commitPendingImportObjectsStep; + this.validatePendingImportObjectsStep = validatePendingImportObjectsStep; this.statusService = statusService; this.experimentPhenotypeService = experimentPhenotypeService; } @Override - public ImportPreviewResponse process(ImportContext context) { + public ImportPreviewResponse process(ImportContext context) throws Exception { ImportUpload upload = context.getUpload(); boolean commit = context.isCommit(); @@ -91,6 +98,13 @@ public ImportPreviewResponse process(ImportContext context) { ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(context); ProcessContext processContext = populateExistingPendingImportObjectsStep.process(context, phenotypeData); ProcessedData processedData = populateNewPendingImportObjectsStep.process(processContext, phenotypeData); + ValidationErrors validationErrors = validatePendingImportObjectsStep.process(context); + + // short circuit if there were validation errors + if (validationErrors.hasErrors()) { + throw new ValidatorException(validationErrors); + } + ImportPreviewResponse response = buildImportPreviewResponse(processedData, upload); statusService.updateMappedData(upload, response, "Finished mapping data to brapi objects"); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java deleted file mode 100644 index 836ae2731..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java +++ /dev/null @@ -1,80 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; - -import io.micronaut.context.annotation.Property; -import org.apache.commons.lang3.StringUtils; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.PendingImportObjectPopulator; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; - -import java.math.BigInteger; -import java.util.Map; -import java.util.UUID; -import java.util.function.Supplier; - -import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.MULTIPLE_EXP_TITLES; -import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.PREEXISTING_EXPERIMENT_TITLE; - -public class CreatePendingImportPopulator implements PendingImportObjectPopulator { - - @Property(name = "brapi.server.reference-source") - private String BRAPI_REFERENCE_SOURCE; - - @Override - public PendingImportObject populateTrial(ImportContext importContext, - PendingData pendingData, - ExperimentObservation importRow, - Supplier expNextVal) - throws UnprocessableEntityException { - - PendingImportObject trialPio; - Program program = importContext.getProgram(); - User user = importContext.getUser(); - boolean commit = importContext.isCommit(); - Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); - Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); - - if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { - PendingImportObject envPio; - trialPio = trialByNameNoScope.get(importRow.getExpTitle()); - envPio = studyByNameNoScope.get(importRow.getEnv()); - - // creating new units for existing experiments and environments is not possible - if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && - (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ - throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); - } - } else if (!trialByNameNoScope.isEmpty()) { - throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); - } else { - UUID id = UUID.randomUUID(); - String expSeqValue = null; - if (commit) { - expSeqValue = expNextVal.get().toString(); - } - BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); - trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); - // NOTE: moved up a level - //trialByNameNoScope.put(importRow.getExpTitle(), trialPio); - } - - return trialPio; - } - - @Override - public PendingImportObject populateStudy(ImportContext importContext, - Supplier expSeqValue, - ExperimentObservation importRow, - Supplier envNextVal) { - return null; - } - - -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index 44c194cbf..fa02c76ed 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -16,30 +16,92 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; +import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.ProgramLocationService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.Utilities; +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; +@Singleton @Slf4j public class CommitPendingImportObjectsStep { + private final BrAPIListDAO brAPIListDAO; + private final BrAPITrialDAO brapiTrialDAO; + private final BrAPIStudyDAO brAPIStudyDAO; + private final BrAPIObservationDAO brAPIObservationDAO; + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private final ProgramLocationService locationService; + private final OntologyService ontologyService; + + @Inject + public CommitPendingImportObjectsStep(BrAPIListDAO brAPIListDAO, + BrAPITrialDAO brapiTrialDAO, + BrAPIStudyDAO brAPIStudyDAO, + BrAPIObservationDAO brAPIObservationDAO, + BrAPIObservationUnitDAO brAPIObservationUnitDAO, + ProgramLocationService locationService, + OntologyService ontologyService) { + this.brAPIListDAO = brAPIListDAO; + this.brapiTrialDAO = brapiTrialDAO; + this.brAPIStudyDAO = brAPIStudyDAO; + this.brAPIObservationDAO = brAPIObservationDAO; + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + this.locationService = locationService; + this.ontologyService = ontologyService; + } + + // TODO: some common code between workflows here that could be broken out, removed append/update specific code public void process(ProcessContext processContext, ProcessedData processedData) { PendingData pendingData = processContext.getPendingData(); + ImportContext importContext = processContext.getImportContext(); + + ImportUpload upload = importContext.getUpload(); + Program program = importContext.getProgram(); + + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + Map> obsVarDatasetByName = pendingData.getObsVarDatasetByName(); + Map> locationByName = pendingData.getLocationByName(); + Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); + Map> observationByHash = pendingData.getObservationByHash(); List newTrials = ProcessorData.getNewObjects(pendingData.getTrialByNameNoScope()); @@ -68,10 +130,251 @@ public void process(ProcessContext processContext, ProcessedData processedData) List newObservationUnits = ProcessorData.getNewObjects(pendingData.getObservationUnitByNameNoScope()); // filter out observations with no 'value' so they will not be saved - List newObservations = ProcessorData.getNewObjects(this.observationByHash) + List newObservations = ProcessorData.getNewObjects(observationByHash) .stream() .filter(obs -> !obs.getValue().isBlank()) .collect(Collectors.toList()); + AuthenticatedUser actingUser = new AuthenticatedUser(upload.getUpdatedByUser().getName(), new ArrayList<>(), upload.getUpdatedByUser().getId(), new ArrayList<>()); + + try { + List createdDatasets = new ArrayList<>(brAPIListDAO.createBrAPILists(newDatasetRequests, program.getId(), upload)); + createdDatasets.forEach(summary -> obsVarDatasetByName.get(summary.getListName()).getBrAPIObject().setListDbId(summary.getListDbId())); + + List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newTrials, program.getId(), upload)); + // set the DbId to the for each newly created trial + for (BrAPITrial createdTrial : createdTrials) { + String createdTrialName = Utilities.removeProgramKey(createdTrial.getTrialName(), program.getKey()); + trialByNameNoScope.get(createdTrialName) + .getBrAPIObject() + .setTrialDbId(createdTrial.getTrialDbId()); + } + + List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); + // set the DbId to the for each newly created location + for (ProgramLocation createdLocation : createdLocations) { + String createdLocationName = createdLocation.getName(); + locationByName.get(createdLocationName) + .getBrAPIObject() + .setLocationDbId(createdLocation.getLocationDbId()); + } + + updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); + List createdStudies = brAPIStudyDAO.createBrAPIStudies(newStudies, program.getId(), upload); + + // set the DbId to the for each newly created study + for (BrAPIStudy createdStudy : createdStudies) { + String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData(createdStudy.getStudyName(), program.getKey()); + studyByNameNoScope.get(createdStudy_name_no_key) + .getBrAPIObject() + .setStudyDbId(createdStudy.getStudyDbId()); + } + + updateObsUnitDependencyValues(pendingData, program.getKey()); + List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); + + // set the DbId to the for each newly created Observation Unit + for (BrAPIObservationUnit createdObservationUnit : createdObservationUnits) { + // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope + String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getStudyName(), program.getKey()); + String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getObservationUnitName(), program.getKey()); + String createdObsUnit_key = ExperimentUtilities.createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); + observationUnitByNameNoScope.get(createdObsUnit_key) + .getBrAPIObject() + .setObservationUnitDbId(createdObservationUnit.getObservationUnitDbId()); + } + + updateObservationDependencyValues(pendingData, program); + brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); + } catch (ApiException e) { + log.error("Error saving experiment import: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error saving experiment import", e); + throw new InternalServerException(e.getMessage(), e); + } + + // NOTE: removed mutated trials code + + datasetNewDataById.forEach((id, dataset) -> { + try { + List existingObsVarIds = brAPIListDAO.getListById(id, program.getId()).getResult().getData(); + List newObsVarIds = dataset + .getData() + .stream() + .filter(obsVarId -> !existingObsVarIds.contains(obsVarId)).collect(Collectors.toList()); + List obsVarIds = new ArrayList<>(existingObsVarIds); + obsVarIds.addAll(newObsVarIds); + dataset.setData(obsVarIds); + brAPIListDAO.updateBrAPIList(id, dataset, program.getId()); + } catch (ApiException e) { + log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating dataset observation variables: ", e); + throw new InternalServerException(e.getMessage(), e); + } + }); + + // NOTE: removed mutated observations code + + } + + private void updateStudyDependencyValues(PendingData pendingData, Map mappedBrAPIImport, String programKey) { + // update location DbIds in studies for all distinct locations + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + + mappedBrAPIImport.values() + .stream() + .map(PendingImport::getLocation) + .forEach(location -> updateStudyLocationDbId(pendingData, location)); + + // update trial DbIds in studies for all distinct trials + trialByNameNoScope.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(trial -> updateTrialDbId(pendingData, trial, programKey)); } + + private void updateTrialDbId(PendingData pendingData, BrAPITrial trial, String programKey) { + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + + studyByNameNoScope.values() + .stream() + .filter(study -> study.getBrAPIObject() + .getTrialName() + .equals(Utilities.removeProgramKey(trial.getTrialName(), programKey))) + .forEach(study -> study.getBrAPIObject() + .setTrialDbId(trial.getTrialDbId())); + } + + private void updateStudyLocationDbId(PendingData pendingData, PendingImportObject location) { + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + + studyByNameNoScope.values() + .stream() + .filter(study -> location.getId().toString() + .equals(study.getBrAPIObject() + .getLocationDbId())) + .forEach(study -> study.getBrAPIObject() + .setLocationDbId(location.getBrAPIObject().getLocationDbId())); + } + + private void updateObsUnitDependencyValues(PendingData pendingData, String programKey) { + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + Map> existingGermplasmByGID = pendingData.getExistingGermplasmByGID(); + + // update study DbIds + studyByNameNoScope.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(study -> updateStudyDbId(pendingData, study, programKey)); + + // update germplasm DbIds + existingGermplasmByGID.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(germplasm -> updateGermplasmDbId(pendingData, germplasm)); + } + + private void updateGermplasmDbId(PendingData pendingData, BrAPIGermplasm germplasm) { + Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); + + observationUnitByNameNoScope.values() + .stream() + .filter(obsUnit -> germplasm.getAccessionNumber() != null && + germplasm.getAccessionNumber().equals(obsUnit + .getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.GID).getAsString())) + .forEach(obsUnit -> obsUnit.getBrAPIObject() + .setGermplasmDbId(germplasm.getGermplasmDbId())); + } + + private void updateStudyDbId(PendingData pendingData, BrAPIStudy study, String programKey) { + Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); + + observationUnitByNameNoScope.values() + .stream() + .filter(obsUnit -> obsUnit.getBrAPIObject() + .getStudyName() + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), programKey))) + .forEach(obsUnit -> { + obsUnit.getBrAPIObject() + .setStudyDbId(study.getStudyDbId()); + obsUnit.getBrAPIObject() + .setTrialDbId(study.getTrialDbId()); + }); + } + + + private void updateObservationDependencyValues(PendingData pendingData, Program program) { + String programKey = program.getKey(); + Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); + Map> observationByHash = pendingData.getObservationByHash(); + + // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds + observationUnitByNameNoScope.values().stream() + .map(PendingImportObject::getBrAPIObject) + .forEach(obsUnit -> updateObservationDbIds(pendingData, obsUnit, programKey)); + + // Update ObservationVariable DbIds + List traits = getTraitList(program); + CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); + for ( Trait trait: traits) { + traitMap.put(trait.getObservationVariableName(),trait); + } + for (PendingImportObject observation : observationByHash.values()) { + String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); + if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { + String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); + observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); + } + } + } + + // Update each ovservation's observationUnit DbId, study DbId, and germplasm DbId + private void updateObservationDbIds(PendingData pendingData, BrAPIObservationUnit obsUnit, String programKey) { + Map> observationByHash = pendingData.getObservationByHash(); + + // FILTER LOGIC: Match on Env and Exp Unit ID + observationByHash.values() + .stream() + .filter(obs -> obs.getBrAPIObject() + .getAdditionalInfo() != null + && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null + && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) + .getAsString() + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getStudyName(), programKey)) + && Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getBrAPIObject().getObservationUnitName(), programKey) + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getObservationUnitName(), programKey)) + ) + .forEach(obs -> { + if (StringUtils.isBlank(obs.getBrAPIObject().getObservationUnitDbId())) { + obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); + } + obs.getBrAPIObject().setStudyDbId(obsUnit.getStudyDbId()); + obs.getBrAPIObject().setGermplasmDbId(obsUnit.getGermplasmDbId()); + }); + } + + private List getTraitList(Program program) { + try { + return ontologyService.getTraitsByProgramId(program.getId(), true); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + throw new InternalServerException(e.toString(), e); + } + } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index d3f83d45b..22799a025 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -207,19 +207,13 @@ private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservatio ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); - observationUnitByName.put(createObservationUnitKey(row), + observationUnitByName.put(ExperimentUtilities.createObservationUnitKey(row), new PendingImportObject<>(ImportObjectState.EXISTING, brAPIObservationUnit, UUID.fromString(idRef.getReferenceId()))); } - private String createObservationUnitKey(ExperimentObservation importRow) { - return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); - } - private String createObservationUnitKey(String studyName, String obsUnitName) { - return studyName + obsUnitName; - } /** * Initializes studies by name without scope. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index cd4faf164..2d22895fc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -16,14 +16,17 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; +import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpStatus; import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.xmlbeans.impl.xb.xsdschema.ImportDocument; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; @@ -36,11 +39,11 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingImportObjectData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.PendingImportObjectPopulator; import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentSeasonService; import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentValidateService; import org.breedinginsight.model.Program; @@ -75,6 +78,9 @@ public class PopulateNewPendingImportObjectsStep { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final DSLContext dsl; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + @Inject public PopulateNewPendingImportObjectsStep(ExperimentValidateService experimentValidateService, ExperimentSeasonService experimentSeasonService, @@ -88,12 +94,14 @@ public PopulateNewPendingImportObjectsStep(ExperimentValidateService experimentV this.dsl = dsl; } - public ProcessedData process(ProcessContext context, ProcessedPhenotypeData phenotypeData) { + public ProcessedData process(ProcessContext context, ProcessedPhenotypeData phenotypeData) + throws MissingRequiredInfoException, UnprocessableEntityException, ApiException { Table data = context.getImportContext().getData(); ImportUpload upload = context.getImportContext().getUpload(); ImportContext importContext = context.getImportContext(); + populatePendingImportObjects(context, phenotypeData); // TODO: implement @@ -101,15 +109,18 @@ public ProcessedData process(ProcessContext context, ProcessedPhenotypeData phen } - // initNew + // NOTE: was called initNew + // initNewBrapiData(importRows, phenotypeCols, program, user, referencedTraits, commit); // TODO: move to shared service - private void populatePendingImportObjects(ImportContext importContext, - ProcessedPhenotypeData phenotypeData, - PendingImportObjectPopulator pioPopulator) { + private void populatePendingImportObjects(ProcessContext processContext, + ProcessedPhenotypeData phenotypeData) + throws MissingRequiredInfoException, UnprocessableEntityException, ApiException { + ImportContext importContext = processContext.getImportContext(); List importRows = importContext.getImportRows(); Program program = importContext.getProgram(); boolean commit = importContext.isCommit(); + PendingData pendingData = processContext.getPendingData(); Supplier expNextVal = getNextExperimentSequenceNumber(program); Supplier envNextVal = getNextEnvironmentSequenceNumber(program); @@ -120,7 +131,7 @@ private void populatePendingImportObjects(ImportContext importContext, for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - populateIndependentVariablePIOsForRow(); + populateIndependentVariablePIOsForRow(importContext, phenotypeData, pendingData, importRow, expNextVal, envNextVal); // ... (Common logic from the original method) @@ -130,7 +141,7 @@ private void populatePendingImportObjects(ImportContext importContext, } } - + // TODO: these sequence methods could be moved to common area /** * Returns a Supplier that generates the next experiment sequence number based on the given Program. * @@ -172,19 +183,18 @@ private Supplier getNextEnvironmentSequenceNumber(Program program) { * @param importRow The import row. * @param expNextVal The supplier for generating experiment next value. * @param envNextVal The supplier for generating environment next value. - * @param pioPopulator The pending import object populator. * @return The populated independent variable PendingImportObjectData. * @throws MissingRequiredInfoException If any required information is missing. * @throws UnprocessableEntityException If the entity is unprocessable. * @throws ApiException If there is an API exception. */ + // TODO: this could potentially be made reusable between workflows in the future private PendingImportObjectData populateIndependentVariablePIOsForRow(ImportContext importContext, ProcessedPhenotypeData phenotypeData, PendingData pendingData, ExperimentObservation importRow, Supplier expNextVal, - Supplier envNextVal, - PendingImportObjectPopulator pioPopulator) + Supplier envNextVal) throws MissingRequiredInfoException, UnprocessableEntityException, ApiException { Program program = importContext.getProgram(); @@ -194,9 +204,9 @@ private PendingImportObjectData populateIndependentVariablePIOsForRow(ImportCont PendingImportObject trialPIO = null; try { - trialPIO = pioPopulator.populateTrial(importContext, pendingData, importRow, expNextVal); + trialPIO = populateTrial(importContext, pendingData, importRow, expNextVal); - // moved up a level + // NOTE: moved up a level if (trialPIO.getState() == ImportObjectState.NEW) { pendingData.getTrialByNameNoScope().put(importRow.getExpTitle(), trialPIO); } @@ -210,15 +220,15 @@ private PendingImportObjectData populateIndependentVariablePIOsForRow(ImportCont .getAdditionalInfo() .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) .getAsString(); - } - if (commit) { - fetchOrCreateDatasetPIO(importRow, program, referencedTraits); + // updates pendingData obsVarDatasetByName PIO + fetchOrCreateDatasetPIO(importContext, pendingData, importRow, referencedTraits); } - fetchOrCreateLocationPIO(importRow); + // updates pendingData locationByName PIO + fetchOrCreateLocationPIO(pendingData, importRow); - PendingImportObject studyPIO = fetchOrCreateStudyPIO(program, commit, expSeqValue, importRow, envNextVal); + PendingImportObject studyPIO = fetchOrCreateStudyPIO(importContext, pendingData, expSeqValue, importRow, envNextVal); String envSeqValue = null; if (commit) { @@ -228,7 +238,7 @@ private PendingImportObjectData populateIndependentVariablePIOsForRow(ImportCont .getAsString(); } - PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); + PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(importContext, pendingData, envSeqValue, importRow); return PendingImportObjectData.builder() .trialPIO(trialPIO) @@ -302,98 +312,67 @@ private void initNewBrapiData(ImportContext importContext, ProcessedPhenotypeDat } } - private Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { - Set ouDbIds = new HashSet<>(); - Set variableDbIds = new HashSet<>(); - Map variableNameByDbId = new HashMap<>(); - Map ouNameByDbId = new HashMap<>(); - Map studyNameByDbId = studyByNameNoScope.values() - .stream() - .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) - .map(PendingImportObject::getBrAPIObject) - .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); - - studyNameByDbId.keySet().forEach(studyDbId -> { - try { - brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { - if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { - ouDbIds.add(ou.getObservationUnitDbId()); - } - ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); - }); - } catch (ApiException e) { - throw new RuntimeException(e); - } - }); + public PendingImportObject populateTrial(ImportContext importContext, + PendingData pendingData, + ExperimentObservation importRow, + Supplier expNextVal) + throws UnprocessableEntityException { - for (Trait referencedTrait : referencedTraits) { - variableDbIds.add(referencedTrait.getObservationVariableDbId()); - variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); - } - - List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); - - return existingObservations.stream() - .map(obs -> { - String studyName = studyNameByDbId.get(obs.getStudyDbId()); - String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); - String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); - - String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); - - return Map.entry(key, obs); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - private PendingImportObject fetchOrCreateTrialPIO( - Program program, - User user, - boolean commit, - ExperimentObservation importRow, - Supplier expNextVal - ) throws UnprocessableEntityException { PendingImportObject trialPio; - - // use the prior trial if observation unit IDs are supplied - // TODO: handle multiple workflows - if (hasAllReferenceUnitIds) { - trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); - - // otherwise create a new trial, but there can be only one allowed + Program program = importContext.getProgram(); + User user = importContext.getUser(); + boolean commit = importContext.isCommit(); + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + + if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { + PendingImportObject envPio; + trialPio = trialByNameNoScope.get(importRow.getExpTitle()); + envPio = studyByNameNoScope.get(importRow.getEnv()); + + // creating new units for existing experiments and environments is not possible + if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && + (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ + throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); + } + } else if (!trialByNameNoScope.isEmpty()) { + throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); } else { - if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { - PendingImportObject envPio; - trialPio = trialByNameNoScope.get(importRow.getExpTitle()); - envPio = studyByNameNoScope.get(importRow.getEnv()); - - // creating new units for existing experiments and environments is not possible - if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && - (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ - throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); - } - } else if (!trialByNameNoScope.isEmpty()) { - throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); - } else { - UUID id = UUID.randomUUID(); - String expSeqValue = null; - if (commit) { - expSeqValue = expNextVal.get().toString(); - } - BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); - trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); - trialByNameNoScope.put(importRow.getExpTitle(), trialPio); + UUID id = UUID.randomUUID(); + String expSeqValue = null; + if (commit) { + expSeqValue = expNextVal.get().toString(); } + BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); + trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); + // NOTE: moved up a level + //trialByNameNoScope.put(importRow.getExpTitle(), trialPio); } + return trialPio; } - private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) throws UnprocessableEntityException { + /** + * Fetches or creates a dataset for import. + * + * @param importContext The import context + * @param pendingData The pending data containing information about the import (modified in place) + * @param importRow The import row representing an observation + * @param referencedTraits The list of referenced Trait objects + * @throws UnprocessableEntityException if the import data is invalid + */ + public void fetchOrCreateDatasetPIO(ImportContext importContext, + PendingData pendingData, + ExperimentObservation importRow, + List referencedTraits) throws UnprocessableEntityException { PendingImportObject pio; + Program program = importContext.getProgram(); + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + Map> obsVarDatasetByName = pendingData.getObsVarDatasetByName(); - // TODO: multiple workflows - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); + + // TODO: this is common to both workflows String name = String.format("Observation Dataset [%s-%s]", program.getKey(), trialPIO.getBrAPIObject() @@ -420,80 +399,137 @@ private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program pr addObsVarsToDatasetDetails(pio, referencedTraits, program); } - private void fetchOrCreateLocationPIO(ExperimentObservation importRow) { + /** + * Add observation variable IDs to the dataset details of a pending import object. + * + * @param pio The pending import object with BrAPIListDetails (modified in place) + * @param referencedTraits The list of referenced Trait objects + * @param program The Program object + */ + // TODO: common to both workflows + private void addObsVarsToDatasetDetails(PendingImportObject pio, List referencedTraits, Program program) { + BrAPIListDetails details = pio.getBrAPIObject(); + referencedTraits.forEach(trait -> { + String id = Utilities.appendProgramKey(trait.getObservationVariableName(), program.getKey()); + + // TODO - Don't append the key if connected to a brapi service operating with legacy data(no appended program key) + + if (!details.getData().contains(id) && ImportObjectState.EXISTING != pio.getState()) { + details.getData().add(id); + } + if (!details.getData().contains(id) && ImportObjectState.EXISTING == pio.getState()) { + details.getData().add(id); + pio.setState(ImportObjectState.MUTATED); + } + }); + } + + /** + * Fetches or creates a ProgramLocation object for import. + * + * @param pendingData The PendingData object containing information about the import (modified in place) + * @param importRow The ExperimentObservation object representing an observation + */ + public void fetchOrCreateLocationPIO(PendingData pendingData, ExperimentObservation importRow) { PendingImportObject pio; - // TODO: multiple workflows - String envLocationName = hasAllReferenceUnitIds ? - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName() : importRow.getEnvLocation(); + String envLocationName = importRow.getEnvLocation(); + // NOTE: other worklow referenced map specific to itself + Map> locationByName = pendingData.getLocationByName(); + + // TODO: common to both workflows if (!locationByName.containsKey((importRow.getEnvLocation()))) { ProgramLocation newLocation = new ProgramLocation(); newLocation.setName(envLocationName); pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); - this.locationByName.put(envLocationName, pio); + locationByName.put(envLocationName, pio); } } + /** + * Fetches an existing study or creates a new study for the given import row. + * + * @param importContext the import context + * @param pendingData the pending data + * @param expSequenceValue the experiment sequence value + * @param importRow the import row + * @param envNextVal the supplier for generating the next environment ID + * + * @return the pending import object containing the study + * + * @throws UnprocessableEntityException if the study is not processable + */ private PendingImportObject fetchOrCreateStudyPIO( - Program program, - boolean commit, + ImportContext importContext, + PendingData pendingData, String expSequenceValue, ExperimentObservation importRow, Supplier envNextVal ) throws UnprocessableEntityException { + + Program program = importContext.getProgram(); + boolean commit = importContext.isCommit(); + + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + Map> locationByName = pendingData.getLocationByName(); + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + PendingImportObject pio; // TODO: multiple workflows - if (hasAllReferenceUnitIds) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), - program.getKey() - ); - pio = studyByNameNoScope.get(studyName); - if (!commit){ - addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); - } - } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { + if (studyByNameNoScope.containsKey(importRow.getEnv())) { pio = studyByNameNoScope.get(importRow.getEnv()); if (!commit){ - addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + ExperimentUtilities.addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); } } else { - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + // NOTE: specific to this workflow, rest common + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); UUID id = UUID.randomUUID(); BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); - newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies + newStudy.setLocationDbId(locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies // It is assumed that the study has only one season, And that the Years and not // the dbId's are stored in getSeason() list. String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season if (commit) { if(StringUtils.isNotBlank(year)) { - String seasonID = this.yearToSeasonDbId(year, program.getId()); + // TODO: look at if this needs to be cleared across runs + String seasonID = experimentSeasonService.yearToSeasonDbId(year, program.getId()); newStudy.setSeasons(Collections.singletonList(seasonID)); } } else { - addYearToStudyAdditionalInfo(program, newStudy, year); + ExperimentUtilities.addYearToStudyAdditionalInfo(program, newStudy, year); } pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); - this.studyByNameNoScope.put(importRow.getEnv(), pio); + studyByNameNoScope.put(importRow.getEnv(), pio); } return pio; } - private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { + private PendingImportObject fetchOrCreateObsUnitPIO(ImportContext importContext, + PendingData pendingData, + String envSeqValue, + ExperimentObservation importRow) + throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { PendingImportObject pio; - String key = createObservationUnitKey(importRow); - // TODO: multiple workflows - if (hasAllReferenceUnitIds) { - pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); - } else if (observationUnitByNameNoScope.containsKey(key)) { + + Program program = importContext.getProgram(); + boolean commit = importContext.isCommit(); + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); + Map> existingGermplasmByGID = pendingData.getExistingGermplasmByGID(); + + String key = ExperimentUtilities.createObservationUnitKey(importRow); + // NOTE: removed other workflow + + if (observationUnitByNameNoScope.containsKey(key)) { pio = observationUnitByNameNoScope.get(key); } else { String germplasmName = ""; - if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { - germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) + if (existingGermplasmByGID.get(importRow.getGid()) != null) { + germplasmName = existingGermplasmByGID.get(importRow.getGid()) .getBrAPIObject() .getGermplasmName(); } @@ -505,7 +541,7 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra .getAdditionalInfo().getAsJsonObject() .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); } - PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); + PendingImportObject studyPIO = studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); @@ -515,14 +551,14 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); if (matchingOU.isEmpty()) { - throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); + throw new MissingRequiredInfoException(ExperimentUtilities.MISSING_OBS_UNIT_ID_ERROR); } else { pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); } } else { pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); } - this.observationUnitByNameNoScope.put(key, pio); + observationUnitByNameNoScope.put(key, pio); } return pio; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java deleted file mode 100644 index cc81a8498..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java +++ /dev/null @@ -1,597 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; - -import io.micronaut.context.annotation.Prototype; -import io.micronaut.http.HttpStatus; -import io.micronaut.http.exceptions.HttpStatusException; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; -import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingImportObjectData; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.PendingImportObjectPopulator; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentPhenotypeService; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentSeasonService; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentValidateService; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.jooq.DSLContext; -import tech.tablesaw.api.Table; -import org.breedinginsight.model.Trait; - -import javax.inject.Inject; -import java.math.BigInteger; -import java.util.*; -import java.util.function.Supplier; - -@Prototype -@Slf4j -public class ProcessStep implements { - - private final ExperimentValidateService experimentValidateService; - private final ExperimentSeasonService experimentSeasonService; - private final ExperimentPhenotypeService experimentPhenotypeService; - private final BrAPIObservationDAO brAPIObservationDAO; - private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; - private final DSLContext dsl; - - @Inject - public ProcessStep(ExperimentValidateService experimentValidateService, - ExperimentSeasonService experimentSeasonService, - ExperimentPhenotypeService experimentPhenotypeService, - BrAPIObservationDAO brAPIObservationDAO, - BrAPIObservationUnitDAO brAPIObservationUnitDAO, - DSLContext dsl) { - this.experimentValidateService = experimentValidateService; - this.experimentSeasonService = experimentSeasonService; - this.experimentPhenotypeService = experimentPhenotypeService; - this.brAPIObservationDAO = brAPIObservationDAO; - this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; - this.dsl = dsl; - } - - public ProcessedData process(ProcessContext context) { - - Table data = context.getImportContext().getData(); - ImportUpload upload = context.getImportContext().getUpload(); - ImportContext importContext = context.getImportContext(); - - ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(importContext); - - - - // TODO: implement - return new ProcessedData(); - } - - -/* - // initNew - private void populatePendingImportObjects(ImportContext importContext, - ProcessedPhenotypeData phenotypeData, - PendingImportObjectPopulator pioPopulator) { - - List importRows = importContext.getImportRows(); - Program program = importContext.getProgram(); - - Supplier expNextVal = getNextExperimentSequenceNumber(program); - Supplier envNextVal = getNextEnvironmentSequenceNumber(program); - - // TODO: handle this - // existingObsByObsHash = fetchExistingObservations(referencedTraits, program); - - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - - populateIndependentVariablePIOsForRow(); - - // ... (Common logic from the original method) - - PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); - - processObservations(importContext, phenotypeData, importRow, rowNum, commit, importRow.getEnvYear(), obsUnitPIO, studyPIO); - } - } -*/ - - /** - * Returns a Supplier that generates the next experiment sequence number based on the given Program. - * - * @param program the Program for which to generate the next experiment sequence number - * @return a Supplier that generates the next experiment sequence number - * @throws HttpStatusException if the program is not properly configured for observation unit import - */ - private Supplier getNextExperimentSequenceNumber(Program program) { - String expSequenceName = program.getExpSequence(); - if (expSequenceName == null) { - log.error(String.format("Program, %s, is missing a value in the exp sequence column.", program.getName())); - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for observation unit import"); - } - return () -> dsl.nextval(expSequenceName.toLowerCase()); - } - - /** - * Retrieves the next environment sequence number for a given program. - * - * @param program The program for which to get the next environment sequence number. - * @return A Supplier representing a function that generates the next environment sequence number. - * @throws HttpStatusException If the program is not properly configured for environment import. - */ - private Supplier getNextEnvironmentSequenceNumber(Program program) { - String envSequenceName = program.getEnvSequence(); - if (envSequenceName == null) { - log.error(String.format("Program, %s, is missing a value in the env sequence column.", program.getName())); - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for environment import"); - } - return () -> dsl.nextval(envSequenceName.toLowerCase()); - } - - /** - * Populates independent variable PendingImportObjectData for a given row of import data. - * - * @param importContext The import context. - * @param phenotypeData The processed phenotype data. - * @param pendingData The pending data. - * @param importRow The import row. - * @param expNextVal The supplier for generating experiment next value. - * @param envNextVal The supplier for generating environment next value. - * @param pioPopulator The pending import object populator. - * @return The populated independent variable PendingImportObjectData. - * @throws MissingRequiredInfoException If any required information is missing. - * @throws UnprocessableEntityException If the entity is unprocessable. - * @throws ApiException If there is an API exception. - */ - private PendingImportObjectData populateIndependentVariablePIOsForRow(ImportContext importContext, - ProcessedPhenotypeData phenotypeData, - PendingData pendingData, - ExperimentObservation importRow, - Supplier expNextVal, - Supplier envNextVal, - PendingImportObjectPopulator pioPopulator) - throws MissingRequiredInfoException, UnprocessableEntityException, ApiException { - - Program program = importContext.getProgram(); - User user = importContext.getUser(); - boolean commit = importContext.isCommit(); - List referencedTraits = phenotypeData.getReferencedTraits(); - - PendingImportObject trialPIO = null; - try { - trialPIO = pioPopulator.populateTrial(importContext, importRow, expNextVal); - - // moved up a level - if (trialPIO.getState() == ImportObjectState.NEW) { - pendingData.getTrialByNameNoScope().put(importRow.getExpTitle(), trialPIO); - } - } catch (UnprocessableEntityException e) { - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); - } - - String expSeqValue = null; - if (commit) { - expSeqValue = trialPIO.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) - .getAsString(); - } - - if (commit) { - fetchOrCreateDatasetPIO(importRow, program, referencedTraits); - } - - fetchOrCreateLocationPIO(importRow); - - PendingImportObject studyPIO = fetchOrCreateStudyPIO(program, commit, expSeqValue, importRow, envNextVal); - - String envSeqValue = null; - if (commit) { - envSeqValue = studyPIO.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER) - .getAsString(); - } - - PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); - - return PendingImportObjectData.builder() - .trialPIO(trialPIO) - .studyPIO(studyPIO) - .obsUnitPIO(obsUnitPIO) - .build(); - } - -/* - private void processObservations(ImportContext importContext, ProcessedPhenotypeData phenotypeData, ExperimentObservation importRow, - int rowNum, boolean commit, String studyYear, - PendingImportObject obsUnitPIO, - PendingImportObject studyPIO) - throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { - Program program = importContext.getProgram(); - User user = importContext.getUser(); - List> phenotypeCols = phenotypeData.getPhenotypeCols(); - Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); - List referencedTraits = phenotypeData.getReferencedTraits(); - - for (Column column : phenotypeCols) { - // ... (Logic for processing observations) - } - } - - // TODO: move common code out - private void initNewBrapiData(ImportContext importContext, ProcessedPhenotypeData phenotypeData) - throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { - - Program program = importContext.getProgram(); - List importRows = importContext.getImportRows(); - User user = importContext.getUser(); - boolean commit = importContext.isCommit(); - Map existingObsByObsHash; - - List referencedTraits = phenotypeData.getReferencedTraits(); - List> phenotypeCols = phenotypeData.getPhenotypeCols(); - Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); - - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - - - for (Column column : phenotypeCols) { - //If associated timestamp column, add - String dateTimeValue = null; - if (timeStampColByPheno.containsKey(column.name())) { - dateTimeValue = timeStampColByPheno.get(column.name()).getString(rowNum); - //If no timestamp, set to midnight - if (!dateTimeValue.isBlank() && !validDateTimeValue(dateTimeValue)) { - dateTimeValue += MIDNIGHT; - } - } - - // get the study year either referenced from the observation unit or listed explicitly on the import row - // TODO: handle this different workflows - String studyYear = hasAllReferenceUnitIds ? studyPIO.getBrAPIObject().getSeasons().get(0) : importRow.getEnvYear(); - String seasonDbId = sharedSeasonService.yearToSeasonDbId(studyYear, program.getId()); - fetchOrCreateObservationPIO( - program, - user, - importRow, - column, //column.name() gets phenotype name - rowNum, - dateTimeValue, - commit, - seasonDbId, - obsUnitPIO, - studyPIO, - referencedTraits - ); - } - } - } - - private Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { - Set ouDbIds = new HashSet<>(); - Set variableDbIds = new HashSet<>(); - Map variableNameByDbId = new HashMap<>(); - Map ouNameByDbId = new HashMap<>(); - Map studyNameByDbId = studyByNameNoScope.values() - .stream() - .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) - .map(PendingImportObject::getBrAPIObject) - .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); - - studyNameByDbId.keySet().forEach(studyDbId -> { - try { - brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { - if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { - ouDbIds.add(ou.getObservationUnitDbId()); - } - ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); - }); - } catch (ApiException e) { - throw new RuntimeException(e); - } - }); - - for (Trait referencedTrait : referencedTraits) { - variableDbIds.add(referencedTrait.getObservationVariableDbId()); - variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); - } - - List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); - - return existingObservations.stream() - .map(obs -> { - String studyName = studyNameByDbId.get(obs.getStudyDbId()); - String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); - String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); - - String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); - - return Map.entry(key, obs); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - private PendingImportObject fetchOrCreateTrialPIO( - Program program, - User user, - boolean commit, - ExperimentObservation importRow, - Supplier expNextVal - ) throws UnprocessableEntityException { - PendingImportObject trialPio; - - // use the prior trial if observation unit IDs are supplied - // TODO: handle multiple workflows - if (hasAllReferenceUnitIds) { - trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); - - // otherwise create a new trial, but there can be only one allowed - } else { - if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { - PendingImportObject envPio; - trialPio = trialByNameNoScope.get(importRow.getExpTitle()); - envPio = studyByNameNoScope.get(importRow.getEnv()); - - // creating new units for existing experiments and environments is not possible - if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && - (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ - throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); - } - } else if (!trialByNameNoScope.isEmpty()) { - throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); - } else { - UUID id = UUID.randomUUID(); - String expSeqValue = null; - if (commit) { - expSeqValue = expNextVal.get().toString(); - } - BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); - trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); - trialByNameNoScope.put(importRow.getExpTitle(), trialPio); - } - } - return trialPio; - } - - private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) throws UnprocessableEntityException { - PendingImportObject pio; - - // TODO: multiple workflows - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); - String name = String.format("Observation Dataset [%s-%s]", - program.getKey(), - trialPIO.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) - .getAsString()); - if (obsVarDatasetByName.containsKey(name)) { - pio = obsVarDatasetByName.get(name); - } else { - UUID id = UUID.randomUUID(); - BrAPIListDetails newDataset = importRow.constructDatasetDetails( - name, - id, - BRAPI_REFERENCE_SOURCE, - program, - trialPIO.getId().toString()); - pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); - trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); - if (ImportObjectState.EXISTING == trialPIO.getState()) { - trialPIO.setState(ImportObjectState.MUTATED); - } - obsVarDatasetByName.put(name, pio); - } - addObsVarsToDatasetDetails(pio, referencedTraits, program); - } - - private void fetchOrCreateLocationPIO(ExperimentObservation importRow) { - PendingImportObject pio; - // TODO: multiple workflows - String envLocationName = hasAllReferenceUnitIds ? - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName() : importRow.getEnvLocation(); - if (!locationByName.containsKey((importRow.getEnvLocation()))) { - ProgramLocation newLocation = new ProgramLocation(); - newLocation.setName(envLocationName); - pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); - this.locationByName.put(envLocationName, pio); - } - } - - private PendingImportObject fetchOrCreateStudyPIO( - Program program, - boolean commit, - String expSequenceValue, - ExperimentObservation importRow, - Supplier envNextVal - ) throws UnprocessableEntityException { - PendingImportObject pio; - // TODO: multiple workflows - if (hasAllReferenceUnitIds) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), - program.getKey() - ); - pio = studyByNameNoScope.get(studyName); - if (!commit){ - addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); - } - } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { - pio = studyByNameNoScope.get(importRow.getEnv()); - if (!commit){ - addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); - } - } else { - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); - UUID trialID = trialPIO.getId(); - UUID id = UUID.randomUUID(); - BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); - newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies - - // It is assumed that the study has only one season, And that the Years and not - // the dbId's are stored in getSeason() list. - String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season - if (commit) { - if(StringUtils.isNotBlank(year)) { - String seasonID = this.yearToSeasonDbId(year, program.getId()); - newStudy.setSeasons(Collections.singletonList(seasonID)); - } - } else { - addYearToStudyAdditionalInfo(program, newStudy, year); - } - - pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); - this.studyByNameNoScope.put(importRow.getEnv(), pio); - } - return pio; - } - - private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { - PendingImportObject pio; - String key = createObservationUnitKey(importRow); - // TODO: multiple workflows - if (hasAllReferenceUnitIds) { - pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); - } else if (observationUnitByNameNoScope.containsKey(key)) { - pio = observationUnitByNameNoScope.get(key); - } else { - String germplasmName = ""; - if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { - germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) - .getBrAPIObject() - .getGermplasmName(); - } - PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; - UUID trialID = trialPIO.getId(); - UUID datasetId = null; - if (commit) { - datasetId = UUID.fromString(trialPIO.getBrAPIObject() - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); - } - PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); - UUID studyID = studyPIO.getId(); - UUID id = UUID.randomUUID(); - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); - - // check for existing units if this is an existing study - if (studyPIO.getBrAPIObject().getStudyDbId() != null) { - List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); - List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); - if (matchingOU.isEmpty()) { - throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); - } else { - pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); - } - } else { - pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); - } - this.observationUnitByNameNoScope.put(key, pio); - } - return pio; - } - - private void fetchOrCreateObservationPIO(Program program, - User user, - ExperimentObservation importRow, - Column column, - Integer rowNum, - String timeStampValue, - boolean commit, - String seasonDbId, - PendingImportObject obsUnitPIO, - PendingImportObject studyPIO, - List referencedTraits) throws ApiException, UnprocessableEntityException { - PendingImportObject pio; - BrAPIObservation newObservation; - String variableName = column.name(); - String value = column.getString(rowNum); - String key; - - // TODO: multiple workflows - if (hasAllReferenceUnitIds) { - String unitName = obsUnitPIO.getBrAPIObject().getObservationUnitName(); - String studyName = studyPIO.getBrAPIObject().getStudyName(); - key = getObservationHash(studyName + unitName, variableName, studyName); - } else { - key = getImportObservationHash(importRow, variableName); - } - - if (existingObsByObsHash.containsKey(key)) { - if (!isObservationMatched(key, value, column, rowNum)){ - - // prior observation with updated value - newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); - if (!isValueMatched(key, value)){ - newObservation.setValue(value); - } else if (!StringUtils.isBlank(timeStampValue) && !isTimestampMatched(key, timeStampValue)) { - DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; - String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timeStampValue)); - newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); - } - pio = new PendingImportObject<>(ImportObjectState.MUTATED, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); - } else { - - // prior observation - pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(existingObsByObsHash.get(key), BrAPIObservation.class, program)); - } - - observationByHash.put(key, pio); - } else if (!this.observationByHash.containsKey(key)){ - - // new observation - // TODO: multiple workflows - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); - - UUID trialID = trialPIO.getId(); - UUID studyID = studyPIO.getId(); - UUID id = UUID.randomUUID(); - newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject(), commit, program, user, BRAPI_REFERENCE_SOURCE, trialID, studyID, obsUnitPIO.getId(), id); - //NOTE: Can't parse invalid timestamp value, so have to skip if invalid. - // Validation error should be thrown for offending value, but that doesn't happen until later downstream - if (timeStampValue != null && !timeStampValue.isBlank() && (validDateValue(timeStampValue) || validDateTimeValue(timeStampValue))) { - newObservation.setObservationTimeStamp(OffsetDateTime.parse(timeStampValue)); - } - - newObservation.setStudyDbId(studyPIO.getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations - - pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); - this.observationByHash.put(key, pio); - } - } - - */ - -} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java new file mode 100644 index 000000000..4ccdcd868 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java @@ -0,0 +1,464 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.JSON; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentSeasonService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.model.Trait; +import org.breedinginsight.model.User; +import org.breedinginsight.utilities.Utilities; +import tech.tablesaw.columns.Column; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Singleton +@Slf4j +public class ValidatePendingImportObjectsStep { + + private static final String BLANK_FIELD_EXPERIMENT = "Field is blank when creating a new experiment"; + private static final String ENV_LOCATION_MISMATCH = "All locations must be the same for a given environment"; + private static final String BLANK_FIELD_ENV = "Field is blank when creating a new environment"; + private static final String BLANK_FIELD_OBS = "Field is blank when creating new observations"; + private static final String ENV_YEAR_MISMATCH = "All years must be the same for a given environment"; + + private final ExperimentSeasonService experimentSeasonService; + private final Gson gson; + + @Inject + public ValidatePendingImportObjectsStep(ExperimentSeasonService experimentSeasonService) { + this.experimentSeasonService = experimentSeasonService; + this.gson = new JSON().getGson(); + } + + public ValidationErrors process(ImportContext input) { + ValidationErrors validationErrors = new ValidationErrors(); + + + return validationErrors; + + } + + private void prepareDataForValidation(List importRows, + PendingData pendingData, + List> phenotypeCols, + Map mappedBrAPIImport) { + + Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + Map> locationByName = pendingData.getLocationByName(); + Map> obsVarDatasetByName = pendingData.getObsVarDatasetByName(); + Map> existingGermplasmByGID = pendingData.getExistingGermplasmByGID(); + + + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); + List> observations = mappedImportRow.getObservations(); + String observationHash; + + // NOTE: Removed append/update workflow code + mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); + mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); + mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); + mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(ExperimentUtilities.createObservationUnitKey(importRow))); + mappedImportRow.setGermplasm(getGidPIO(pendingData, importRow)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(ExperimentUtilities.getImportObservationHash(importRow, ExperimentUtilities.getVariableNameFromColumn(column)))); + } + + mappedBrAPIImport.put(rowNum, mappedImportRow); + } + } + + private PendingImportObject getGidPIO(PendingData pendingData, ExperimentObservation importRow) { + + Map> existingGermplasmByGID = pendingData.getExistingGermplasmByGID(); + + if (existingGermplasmByGID.containsKey(importRow.getGid())) { + return existingGermplasmByGID.get(importRow.getGid()); + } + + return null; + } + + private ValidationErrors validateFields(List importRows, Map mappedBrAPIImport, List referencedTraits, Program program, + List> phenotypeCols, boolean commit, User user, + PendingData pendingData, + ProcessedPhenotypeData phenotypeData) { + //fetching any existing observations for any OUs in the import + CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); + ValidationErrors validationErrors = new ValidationErrors(); + + for ( Trait trait: referencedTraits) { + colVarMap.put(trait.getObservationVariableName(),trait); + } + Set uniqueStudyAndObsUnit = new HashSet<>(); + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); + // NOTE: validate Observations used by both workflows + if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. + validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); + } + validateTestOrCheck(importRow, validationErrors, rowNum); + validateConditionallyRequired(pendingData, validationErrors, rowNum, importRow, program, commit); + validateObservationUnits(pendingData, validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + validateObservations(pendingData, phenotypeData, validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); + } + + return validationErrors; + } + + private void validateGermplasm(ExperimentObservation importRow, ValidationErrors validationErrors, int rowNum, PendingImportObject germplasmPIO) { + // error if GID is not blank but GID does not already exist + if (StringUtils.isNotBlank(importRow.getGid()) && germplasmPIO == null) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.GERMPLASM_GID, "A non-existing GID", validationErrors, rowNum); + } + } + + private void validateTestOrCheck(ExperimentObservation importRow, ValidationErrors validationErrors, int rowNum) { + String testOrCheck = importRow.getTestOrCheck(); + if ( ! ( testOrCheck==null || testOrCheck.isBlank() + || "C".equalsIgnoreCase(testOrCheck) || "CHECK".equalsIgnoreCase(testOrCheck) + || "T".equalsIgnoreCase(testOrCheck) || "TEST".equalsIgnoreCase(testOrCheck) ) + ){ + ExperimentUtilities.addRowError(ExperimentObservation.Columns.TEST_CHECK, String.format("Invalid value (%s)", testOrCheck), validationErrors, rowNum) ; + } + } + + private void validateConditionallyRequired(PendingData pendingData, ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, Program program, boolean commit) { + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + + ImportObjectState expState = trialByNameNoScope.get(importRow.getExpTitle()) + .getState(); + ImportObjectState envState = studyByNameNoScope.get(importRow.getEnv()).getState(); + + String errorMessage = BLANK_FIELD_EXPERIMENT; + if (expState == ImportObjectState.EXISTING && envState == ImportObjectState.NEW) { + errorMessage = BLANK_FIELD_ENV; + } else if(expState == ImportObjectState.EXISTING && envState == ImportObjectState.EXISTING) { + errorMessage = BLANK_FIELD_OBS; + } + + if(expState == ImportObjectState.NEW || envState == ImportObjectState.NEW) { + validateRequiredCell(importRow.getGid(), ExperimentObservation.Columns.GERMPLASM_GID, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpTitle(), ExperimentObservation.Columns.EXP_TITLE,errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpUnit(), ExperimentObservation.Columns.EXP_UNIT, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpType(), ExperimentObservation.Columns.EXP_TYPE, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getEnv(), ExperimentObservation.Columns.ENV, errorMessage, validationErrors, rowNum); + if(validateRequiredCell(importRow.getEnvLocation(), ExperimentObservation.Columns.ENV_LOCATION, errorMessage, validationErrors, rowNum)) { + if(!Utilities.removeProgramKeyAndUnknownAdditionalData(studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getLocationName(), program.getKey()).equals(importRow.getEnvLocation())) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.ENV_LOCATION, ENV_LOCATION_MISMATCH, validationErrors, rowNum); + } + } + if(validateRequiredCell(importRow.getEnvYear(), ExperimentObservation.Columns.ENV_YEAR, errorMessage, validationErrors, rowNum)) { + String studyYear = StringUtils.defaultString(studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getSeasons().get(0) ); + String rowYear = importRow.getEnvYear(); + if(commit) { + rowYear = experimentSeasonService.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); + } + if(StringUtils.isNotBlank(studyYear) && !studyYear.equals(rowYear)) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.ENV_YEAR, ENV_YEAR_MISMATCH, validationErrors, rowNum); + } + } + validateRequiredCell(importRow.getExpUnitId(), ExperimentObservation.Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpReplicateNo(), ExperimentObservation.Columns.REP_NUM, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpBlockNo(), ExperimentObservation.Columns.BLOCK_NUM, errorMessage, validationErrors, rowNum); + + if(StringUtils.isNotBlank(importRow.getObsUnitID())) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); + } + } else { + //Check if existing environment. If so, ObsUnitId must be assigned + validateRequiredCell( + importRow.getObsUnitID(), + ExperimentObservation.Columns.OBS_UNIT_ID, + ExperimentUtilities.MISSING_OBS_UNIT_ID_ERROR, + validationErrors, + rowNum + ); + } + } + + private boolean validateRequiredCell(String value, String columnHeader, String errorMessage, ValidationErrors validationErrors, int rowNum) { + if (StringUtils.isBlank(value)) { + ExperimentUtilities.addRowError(columnHeader, errorMessage, validationErrors, rowNum); + return false; + } + return true; + } + + private void validateObservationUnits( + PendingData pendingData, + ValidationErrors validationErrors, + Set uniqueStudyAndObsUnit, + int rowNum, + ExperimentObservation importRow) { + Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); + + validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + + String key = ExperimentUtilities.createObservationUnitKey(importRow); + PendingImportObject ouPIO = observationUnitByNameNoScope.get(key); + if(ouPIO.getState() == ImportObjectState.NEW && StringUtils.isNotBlank(importRow.getObsUnitID())) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "Could not find observation unit by ObsUnitDBID", validationErrors, rowNum); + } + + validateGeoCoordinates(validationErrors, rowNum, importRow); + } + + /** + * Validate that the observation unit is unique within a study. + *
+ * SIDE EFFECTS: validationErrors and uniqueStudyAndObsUnit can be modified. + * + * @param validationErrors can be modified as a side effect. + * @param uniqueStudyAndObsUnit can be modified as a side effect. + * @param rowNum counter that is always two less the file row being validated + * @param importRow the data row being validated + */ + private void validateUniqueObsUnits( + ValidationErrors validationErrors, + Set uniqueStudyAndObsUnit, + int rowNum, + ExperimentObservation importRow) { + String envIdPlusStudyId = ExperimentUtilities.createObservationUnitKey(importRow); + if (uniqueStudyAndObsUnit.contains(envIdPlusStudyId)) { + String errorMessage = String.format("The ID (%s) is not unique within the environment(%s)", importRow.getExpUnitId(), importRow.getEnv()); + ExperimentUtilities.addRowError(ExperimentObservation.Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); + } else { + uniqueStudyAndObsUnit.add(envIdPlusStudyId); + } + } + + private void validateGeoCoordinates(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow) { + + String lat = importRow.getLatitude(); + String lon = importRow.getLongitude(); + String elevation = importRow.getElevation(); + + // If any of Lat, Long, or Elevation are provided, Lat and Long must both be provided. + if (StringUtils.isNotBlank(lat) || StringUtils.isNotBlank(lon) || StringUtils.isNotBlank(elevation)) { + if (StringUtils.isBlank(lat)) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.LAT, "Latitude must be provided for complete coordinate specification", validationErrors, rowNum); + } + if (StringUtils.isBlank(lon)) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.LONG, "Longitude must be provided for complete coordinate specification", validationErrors, rowNum); + } + } + + // Validate coordinate values + boolean latBadValue = false; + boolean lonBadValue = false; + boolean elevationBadValue = false; + double latDouble; + double lonDouble; + double elevationDouble; + + // Only check latitude format if not blank since already had previous error + if (StringUtils.isNotBlank(lat)) { + try { + latDouble = Double.parseDouble(lat); + if (latDouble < -90 || latDouble > 90) { + latBadValue = true; + } + } catch (NumberFormatException e) { + latBadValue = true; + } + } + + // Only check longitude format if not blank since already had previous error + if (StringUtils.isNotBlank(lon)) { + try { + lonDouble = Double.parseDouble(lon); + if (lonDouble < -180 || lonDouble > 180) { + lonBadValue = true; + } + } catch (NumberFormatException e) { + lonBadValue = true; + } + } + + if (StringUtils.isNotBlank(elevation)) { + try { + elevationDouble = Double.parseDouble(elevation); + } catch (NumberFormatException e) { + elevationBadValue = true; + } + } + + if (latBadValue) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.LAT, "Invalid Lat value (expected range -90 to 90)", validationErrors, rowNum); + } + + if (lonBadValue) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.LONG, "Invalid Long value (expected range -180 to 180)", validationErrors, rowNum); + } + + if (elevationBadValue) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.LONG, "Invalid Elevation value (numerals expected)", validationErrors, rowNum); + } + + } + + + private void validateObservations(PendingData pendingData, + ProcessedPhenotypeData phenotypeData, + ValidationErrors validationErrors, + int rowNum, + ExperimentObservation importRow, + List> phenotypeCols, + CaseInsensitiveMap colVarMap, + boolean commit, + User user) { + + Map existingObsByObsHash = pendingData.getExistingObsByObsHash(); + Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); + + phenotypeCols.forEach(phenoCol -> { + String importHash; + String importObsValue = phenoCol.getString(rowNum); + + // NOTE: removed append / update specifc code + importHash = ExperimentUtilities.getImportObservationHash(importRow, phenoCol.name()); + + // error if import observation data already exists and user has not selected to overwrite + if(commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && + existingObsByObsHash.containsKey(importHash) && + StringUtils.isNotBlank(phenoCol.getString(rowNum)) && + !existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { + ExperimentUtilities.addRowError( + phenoCol.name(), + String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), + validationErrors, rowNum + ); + + // preview case where observation has already been committed and the import row ObsVar data differs from what + // had been saved prior to import + } else if (existingObsByObsHash.containsKey(importHash) && !ExperimentUtilities.isObservationMatched(phenotypeData, pendingData, importHash, importObsValue, phenoCol, rowNum)) { + + // different data means validations still need to happen + // TODO consider moving these two calls into a separate method since called twice together + ExperimentUtilities.validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); + + //Timestamp validation + if(timeStampColByPheno.containsKey(phenoCol.name())) { + Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); + ExperimentUtilities.validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); + } + + // add a change log entry when updating the value of an observation + // only will update and thereby need change log entry if no error + if (commit && (!validationErrors.hasErrors())) { + BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; + String prior = ""; + if (ExperimentUtilities.isValueMatched(pendingData, importHash, importObsValue)) { + prior.concat(existingObsByObsHash.get(importHash).getValue()); + } + if (timeStampColByPheno.containsKey(phenoCol.name()) && ExperimentUtilities.isTimestampMatched(pendingData, importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { + prior = prior.isEmpty() ? prior : prior.concat(" "); + prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); + } + ChangeLogEntry change = new ChangeLogEntry(prior, + reason, + user.getId(), + timestamp + ); + + // create the changelog field in additional info if it does not already exist + if (pendingObservation.getAdditionalInfo().isJsonNull()) { + pendingObservation.setAdditionalInfo(new JsonObject()); + pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + + if (pendingObservation.getAdditionalInfo() != null && !pendingObservation.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { + pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + + // add a new entry to the changelog + pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); + } + + // preview case where observation has already been committed and import ObsVar data is the + // same as has been committed prior to import + } else if(ExperimentUtilities.isObservationMatched(phenotypeData, pendingData, importHash, importObsValue, phenoCol, rowNum)) { + BrAPIObservation existingObs = existingObsByObsHash.get(importHash); + existingObs.setObservationVariableName(phenoCol.name()); + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + observationByHash.get(importHash).setBrAPIObject(existingObs); + + // preview case where observation has already been committed and import ObsVar data is empty prior to import + } else if(!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + } else { + ExperimentUtilities.validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); + + //Timestamp validation + if(timeStampColByPheno.containsKey(phenoCol.name())) { + Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); + ExperimentUtilities.validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); + } + } + }); + } + + + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java index 5fd67a023..c365cbf6d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java @@ -16,6 +16,8 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment.services; +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; @@ -30,7 +32,8 @@ import java.util.Map; import java.util.UUID; -@Singleton +// reset cache across uses by creating new instance each time this service is injected +@Prototype @Slf4j public class ExperimentSeasonService { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java index af2bcb29e..d12ff2509 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java @@ -113,4 +113,6 @@ private List fetchFileTraits(UUID programId, Collection varNames) throw new InternalServerException(e.toString(), e); } } + + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java deleted file mode 100644 index 9ca689447..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.services.processors.experiment.services; - -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; - -import java.math.BigInteger; -import java.util.function.Supplier; - -public interface PendingImportObjectPopulator { - - PendingImportObject populateTrial( - ImportContext importContext, - PendingData pendingData, - ExperimentObservation importRow, - Supplier expNextVal - ) throws UnprocessableEntityException; - - PendingImportObject populateStudy( - ImportContext importContext, - Supplier expSeqValue, - ExperimentObservation importRow, - Supplier envNextVal); -} From 908742931444bff84004a440673b0ea638067380 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 13 Jun 2024 13:04:39 -0400 Subject: [PATCH 123/203] create entity factory and read factory --- .../appendoverwrite/action/BrAPIAction.java | 4 +- .../action/create/BrAPICreation.java | 9 ++- .../action/create/BrAPIDatasetCreation.java | 7 +- .../create/BrAPIObservationCreation.java | 6 +- .../create/BrAPIObservationUnitCreation.java | 7 +- .../action/create/BrAPIStudyCreation.java | 7 +- .../action/create/BrAPITrialCreation.java | 31 ++++---- .../action/create/LocationCreation.java | 7 +- ...rAPIDatasetReadWorkflowInitialization.java | 7 +- ...PIGermplasmReadWorkflowInitialization.java | 7 +- ...rvationUnitReadWorkflowInitialization.java | 7 +- .../action/read/BrAPIRead.java | 6 +- .../action/read/BrAPIReadFactory.java | 31 ++++++++ .../read/BrAPIReadWorkflowInitialization.java | 13 +++- .../BrAPIStudyReadWorkflowInitialization.java | 8 +- .../action/read/BrAPITrialRead.java | 27 ++++--- .../BrAPITrialReadWorkflowInitialization.java | 26 +++---- .../LocationReadWorkflowInitialization.java | 7 +- .../read/WorkflowReadInitialization.java | 73 +++++++++++++++++++ .../action/update/BrAPIObservationUpdate.java | 7 +- .../action/update/BrAPITrialUpdate.java | 28 ++++--- .../action/update/BrAPIUpdate.java | 11 ++- .../entity/PendingEntityFactory.java | 40 ++++++++++ .../appendoverwrite/entity/PendingTrial.java | 24 ++++-- .../entity/PendingTrialProvider.java | 46 ++++++++++++ .../middleware/commit/BrAPITrialCommit.java | 9 ++- .../initialize/WorkflowInitialization.java | 14 +++- .../RequiredBrAPIData.java | 2 +- .../RequiredDatasets.java | 2 +- .../RequiredGermplasm.java | 2 +- .../RequiredLocations.java | 2 +- .../RequiredObservationUnits.java | 2 +- .../{initialize => misc}/RequiredStudies.java | 2 +- .../{initialize => misc}/RequiredTrials.java | 8 +- .../utilities/cache/RedissonManager.java | 26 +++++++ src/main/resources/application.yml | 1 + 36 files changed, 393 insertions(+), 123 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/WorkflowReadInitialization.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrialProvider.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{initialize => misc}/RequiredBrAPIData.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{initialize => misc}/RequiredDatasets.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{initialize => misc}/RequiredGermplasm.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{initialize => misc}/RequiredLocations.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{initialize => misc}/RequiredObservationUnits.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{initialize => misc}/RequiredStudies.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{initialize => misc}/RequiredTrials.java (83%) create mode 100644 src/main/java/org/breedinginsight/utilities/cache/RedissonManager.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIAction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIAction.java index 318337df3..2c00f8c17 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIAction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIAction.java @@ -2,7 +2,6 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import java.util.Optional; @@ -27,10 +26,9 @@ public interface BrAPIAction { /** * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. * - * @param context The ExpUnitMiddlewareContext providing information about the entity. * @return The ExperimentImportEntity representing the BrAPI entity being acted on. */ - ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context); + ExperimentImportEntity getEntity(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java index 063781a79..2def125e1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java @@ -30,15 +30,18 @@ public abstract class BrAPICreation implements BrAPIAction { * @param context the ExpUnitMiddlewareContext object */ protected BrAPICreation(ExpUnitMiddlewareContext context) { - this.entity = getEntity(context); + this.entity = getEntity(); + } + + protected BrAPICreation() { } /** * Abstract method to get the ExperimentImportEntity based on the ExpUnitMiddlewareContext. - * @param context the ExpUnitMiddlewareContext object + * * @return the ExperimentImportEntity object */ - public abstract ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context); + public abstract ExperimentImportEntity getEntity(); /** * Executes the creation process for entities. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java index 5b2026a0d..b22f7c447 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java @@ -12,23 +12,24 @@ @Slf4j public class BrAPIDatasetCreation extends BrAPICreation { + ExpUnitMiddlewareContext context; /** * Constructor for BrAPICreation class. * * @param context the ExpUnitMiddlewareContext object */ public BrAPIDatasetCreation(ExpUnitMiddlewareContext context) { - super(context); + + this.context = context; } /** * Abstract method to get the ExperimentImportEntity based on the ExpUnitMiddlewareContext. * - * @param context the ExpUnitMiddlewareContext object * @return the ExperimentImportEntity object */ @Override - public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + public ExperimentImportEntity getEntity() { try (ApplicationContext appContext = ApplicationContext.run()) { BrAPIListDAO brAPIListDAO = appContext.getBean(BrAPIListDAO.class); DatasetService datasetService = appContext.getBean(DatasetService.class); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationCreation.java index e204068b6..d034d09a4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationCreation.java @@ -13,12 +13,14 @@ @Slf4j public class BrAPIObservationCreation extends BrAPICreation { + ExpUnitMiddlewareContext context; public BrAPIObservationCreation(ExpUnitMiddlewareContext context) { - super(context); + + this.context = context; } @Override - public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + public ExperimentImportEntity getEntity() { try (ApplicationContext appContext = ApplicationContext.run()) { BrAPIObservationDAO brAPIObservationDAO = appContext.getBean(BrAPIObservationDAO.class); OntologyService ontologyService = appContext.getBean(OntologyService.class); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationUnitCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationUnitCreation.java index 21e492415..08ab6311c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationUnitCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationUnitCreation.java @@ -10,23 +10,24 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; public class BrAPIObservationUnitCreation extends BrAPICreation { + ExpUnitMiddlewareContext context; /** * Constructor for BrAPICreation class. * * @param context the ExpUnitMiddlewareContext object */ public BrAPIObservationUnitCreation(ExpUnitMiddlewareContext context) { - super(context); + + this.context = context; } /** * Abstract method to get the ExperimentImportEntity based on the ExpUnitMiddlewareContext. * - * @param context the ExpUnitMiddlewareContext object * @return the ExperimentImportEntity object */ @Override - public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + public ExperimentImportEntity getEntity() { try (ApplicationContext appContext = ApplicationContext.run()) { BrAPIObservationUnitDAO observationUnitDAO = appContext.getBean(BrAPIObservationUnitDAO.class); ObservationUnitService observationUnitService = appContext.getBean(ObservationUnitService.class); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIStudyCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIStudyCreation.java index 1020c0402..6995b5731 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIStudyCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIStudyCreation.java @@ -13,18 +13,19 @@ @Slf4j public class BrAPIStudyCreation extends BrAPICreation { + ExpUnitMiddlewareContext context; public BrAPIStudyCreation(ExpUnitMiddlewareContext context) { - super(context); + + this.context = context; } /** * Abstract method to get the ExperimentImportEntity based on the ExpUnitMiddlewareContext. * - * @param context the ExpUnitMiddlewareContext object * @return the ExperimentImportEntity object */ @Override - public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + public ExperimentImportEntity getEntity() { try (ApplicationContext appContext = ApplicationContext.run()) { StudyService studyService = appContext.getBean(StudyService.class); BrAPIStudyDAO brAPIStudyDAO = appContext.getBean(BrAPIStudyDAO.class); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPITrialCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPITrialCreation.java index 71fa1ba14..b6bba6e7b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPITrialCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPITrialCreation.java @@ -1,30 +1,31 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; -import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingTrial; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; @Slf4j public class BrAPITrialCreation extends BrAPICreation { + ExpUnitMiddlewareContext context; + private PendingEntityFactory pendingEntityFactory; + public BrAPITrialCreation(ExpUnitMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { - public BrAPITrialCreation(ExpUnitMiddlewareContext context) { - super(context); + this.context = context; + this.pendingEntityFactory = pendingEntityFactory; } @Override - public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - try (ApplicationContext appContext = ApplicationContext.run()) { - TrialService trialService = appContext.getBean(TrialService.class); - BrAPITrialDAO brAPITrialDAO = appContext.getBean(BrAPITrialDAO.class); - ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); - - return new PendingTrial(context, trialService, brAPITrialDAO, experimentUtilities); - } + public ExperimentImportEntity getEntity() { +// try (ApplicationContext appContext = ApplicationContext.run()) { +// TrialService trialService = appContext.getBean(TrialService.class); +// BrAPITrialDAO brAPITrialDAO = appContext.getBean(BrAPITrialDAO.class); +// ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); +// +// return new PendingTrial(context, trialService, brAPITrialDAO, experimentUtilities); +// } + return pendingEntityFactory.pendingTrialBean(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/LocationCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/LocationCreation.java index 9ae5167ee..9a18bbef5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/LocationCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/LocationCreation.java @@ -10,23 +10,24 @@ import org.breedinginsight.services.ProgramLocationService; public class LocationCreation extends BrAPICreation{ + ExpUnitMiddlewareContext context; /** * Constructor for BrAPICreation class. * * @param context the ExpUnitMiddlewareContext object */ public LocationCreation(ExpUnitMiddlewareContext context) { - super(context); + + this.context = context; } /** * Abstract method to get the ExperimentImportEntity based on the ExpUnitMiddlewareContext. * - * @param context the ExpUnitMiddlewareContext object * @return the ExperimentImportEntity object */ @Override - public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + public ExperimentImportEntity getEntity() { try (ApplicationContext appContext = ApplicationContext.run()) { ProgramLocationService programLocationService = appContext.getBean(ProgramLocationService.class); LocationService locationService = appContext.getBean(LocationService.class); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java index d6d3f9f69..ca51d1a46 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java @@ -6,6 +6,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingDataset; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; @@ -16,18 +17,18 @@ public class BrAPIDatasetReadWorkflowInitialization extends BrAPIReadWorkflowIni * * @param context the ExpUnitMiddlewareContext used for initialization. */ + ExpUnitMiddlewareContext context; public BrAPIDatasetReadWorkflowInitialization(ExpUnitMiddlewareContext context) { - super(context); + this.context = context; } /** * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. * - * @param context The ExpUnitMiddlewareContext providing information about the entity. * @return The ExperimentImportEntity representing the BrAPI entity being acted on. */ @Override - public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + public ExperimentImportEntity getEntity() { try (ApplicationContext appContext = ApplicationContext.run()) { BrAPIListDAO brAPIListDAO = appContext.getBean(BrAPIListDAO.class); DatasetService datasetService = appContext.getBean(DatasetService.class); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java index b60c8735d..351ef620d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java @@ -4,11 +4,13 @@ import org.brapi.v2.model.germ.BrAPIGermplasm; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingGermplasm; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.GermplasmService; public class BrAPIGermplasmReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { + ExpUnitMiddlewareContext context; /** * Constructs a new BrAPIReadWorkflowInitialization object with the given ExpUnitMiddlewareContext. * Initializes the entity based on the provided context. @@ -16,17 +18,16 @@ public class BrAPIGermplasmReadWorkflowInitialization extends BrAPIReadWorkflowI * @param context the ExpUnitMiddlewareContext used for initialization. */ public BrAPIGermplasmReadWorkflowInitialization(ExpUnitMiddlewareContext context) { - super(context); + this.context = context; } /** * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. * - * @param context The ExpUnitMiddlewareContext providing information about the entity. * @return The ExperimentImportEntity representing the BrAPI entity being acted on. */ @Override - public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + public ExperimentImportEntity getEntity() { try (ApplicationContext appContext = ApplicationContext.run()) { GermplasmService germplasmService = appContext.getBean(GermplasmService.class); ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java index 6bf97b8a9..f124edd6e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java @@ -5,11 +5,13 @@ import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; public class BrAPIObservationUnitReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { + ExpUnitMiddlewareContext context; /** * Constructs a new BrAPIReadWorkflowInitialization object with the given ExpUnitMiddlewareContext. * Initializes the entity based on the provided context. @@ -17,17 +19,16 @@ public class BrAPIObservationUnitReadWorkflowInitialization extends BrAPIReadWor * @param context the ExpUnitMiddlewareContext used for initialization. */ public BrAPIObservationUnitReadWorkflowInitialization(ExpUnitMiddlewareContext context) { - super(context); + this.context = context; } /** * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. * - * @param context The ExpUnitMiddlewareContext providing information about the entity. * @return The ExperimentImportEntity representing the BrAPI entity being acted on. */ @Override - public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + public ExperimentImportEntity getEntity() { try (ApplicationContext appContext = ApplicationContext.run()) { BrAPIObservationUnitDAO observationUnitDAO = appContext.getBean(BrAPIObservationUnitDAO.class); ObservationUnitService observationUnitService = appContext.getBean(ObservationUnitService.class); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIRead.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIRead.java index 3515e9dec..f3901743c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIRead.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIRead.java @@ -17,8 +17,12 @@ public abstract class BrAPIRead implements BrAPIAction { ExperimentImportEntity entity; protected BrAPIRead(ExpUnitMiddlewareContext context) { - this.entity = getEntity(context); + this.entity = getEntity(); } + + protected BrAPIRead() { + } + public Optional execute() throws ApiException { try { List fetchedMembers = entity.brapiRead(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java new file mode 100644 index 000000000..447969a92 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java @@ -0,0 +1,31 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; + +import io.micronaut.context.annotation.Bean; +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Prototype; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +import javax.inject.Inject; + +@Factory +public class BrAPIReadFactory { + private final PendingEntityFactory pendingEntityFactory; + + @Inject + public BrAPIReadFactory(PendingEntityFactory pendingEntityFactory) { + this.pendingEntityFactory = pendingEntityFactory; + } + + public static WorkflowReadInitialization trialWorkflowReadInitialization(ExpUnitMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowReadInitialization(pendingEntityFactory.pendingTrialBean(context)); + } + + @Bean + @Prototype + public WorkflowReadInitialization trialWorkflowReadInitialization(ExpUnitMiddlewareContext context) { + return trialWorkflowReadInitialization(context, pendingEntityFactory); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java index f11e1db3e..04f00b824 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java @@ -7,9 +7,11 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.utilities.Utilities; +import javax.inject.Inject; import java.util.List; import java.util.Optional; @@ -23,6 +25,7 @@ public abstract class BrAPIReadWorkflowInitialization implements BrAPIAction { ExperimentImportEntity entity; // The entity used for read operations initialization + PendingEntityFactory pendingEntityFactory; /** * Constructs a new BrAPIReadWorkflowInitialization object with the given ExpUnitMiddlewareContext. @@ -30,8 +33,14 @@ public abstract class BrAPIReadWorkflowInitialization implements BrAPIAction< * * @param context the ExpUnitMiddlewareContext used for initialization. */ - protected BrAPIReadWorkflowInitialization(ExpUnitMiddlewareContext context) { - this.entity = getEntity(context); + @Inject + protected BrAPIReadWorkflowInitialization(PendingEntityFactory pendingEntityFactory) { + this.entity = getEntity(); + this.pendingEntityFactory = pendingEntityFactory; + } + + @Inject + protected BrAPIReadWorkflowInitialization() { } /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java index 3d75fc285..6f48b0ce1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java @@ -5,24 +5,24 @@ import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingStudy; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; public class BrAPIStudyReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { - + ExpUnitMiddlewareContext context; public BrAPIStudyReadWorkflowInitialization(ExpUnitMiddlewareContext context) { - super(context); + this.context = context; } /** * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. * - * @param context The ExpUnitMiddlewareContext providing information about the entity. * @return The ExperimentImportEntity representing the BrAPI entity being acted on. */ @Override - public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + public ExperimentImportEntity getEntity() { try (ApplicationContext appContext = ApplicationContext.run()) { StudyService studyService = appContext.getBean(StudyService.class); BrAPIStudyDAO brAPIStudyDAO = appContext.getBean(BrAPIStudyDAO.class); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialRead.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialRead.java index 47e484e48..628c78b68 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialRead.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialRead.java @@ -1,30 +1,29 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; -import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingTrial; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; @Slf4j public class BrAPITrialRead extends BrAPIRead { + ExpUnitMiddlewareContext context; public BrAPITrialRead(ExpUnitMiddlewareContext context) { - super(context); + + this.context = context; } @Override - public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - try (ApplicationContext appContext = ApplicationContext.run()) { - TrialService trialService = appContext.getBean(TrialService.class); - BrAPITrialDAO brAPITrialDAO = appContext.getBean(BrAPITrialDAO.class); - ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); - - return new PendingTrial(context, trialService, brAPITrialDAO, experimentUtilities); - } + public ExperimentImportEntity getEntity() { +// try (ApplicationContext appContext = ApplicationContext.run()) { +// TrialService trialService = appContext.getBean(TrialService.class); +// BrAPITrialDAO brAPITrialDAO = appContext.getBean(BrAPITrialDAO.class); +// ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); +// +// return new PendingTrial(context, trialService, brAPITrialDAO, experimentUtilities); +// } + return null; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialReadWorkflowInitialization.java index e7a49b418..342c727f8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialReadWorkflowInitialization.java @@ -1,31 +1,29 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; -import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.BrAPIReadWorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingTrial; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; @Slf4j public class BrAPITrialReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { + ExpUnitMiddlewareContext context; + public BrAPITrialReadWorkflowInitialization(ExpUnitMiddlewareContext context) { - super(context); + this.context = context; } @Override - public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - try (ApplicationContext appContext = ApplicationContext.run()) { - TrialService trialService = appContext.getBean(TrialService.class); - BrAPITrialDAO brAPITrialDAO = appContext.getBean(BrAPITrialDAO.class); - ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); + public ExperimentImportEntity getEntity() { - return new PendingTrial(context, trialService, brAPITrialDAO, experimentUtilities); - } + return pendingEntityFactory.pendingTrialBean(context); +// try (ApplicationContext appContext = ApplicationContext.run()) { +// TrialService trialService = appContext.getBean(TrialService.class); +// BrAPITrialDAO brAPITrialDAO = appContext.getBean(BrAPITrialDAO.class); +// ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); +// +// return new PendingTrial(context, trialService, brAPITrialDAO, experimentUtilities); +// } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java index e0196d8d8..fbac2e0fd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java @@ -4,6 +4,7 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingLocation; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; @@ -12,6 +13,7 @@ @Slf4j public class LocationReadWorkflowInitialization extends BrAPIReadWorkflowInitialization{ + ExpUnitMiddlewareContext context; /** * Constructs a new BrAPIReadWorkflowInitialization object with the given ExpUnitMiddlewareContext. * Initializes the entity based on the provided context. @@ -19,17 +21,16 @@ public class LocationReadWorkflowInitialization extends BrAPIReadWorkflowInitial * @param context the ExpUnitMiddlewareContext used for initialization. */ public LocationReadWorkflowInitialization(ExpUnitMiddlewareContext context) { - super(context); + this.context = context; } /** * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. * - * @param context The ExpUnitMiddlewareContext providing information about the entity. * @return The ExperimentImportEntity representing the BrAPI entity being acted on. */ @Override - public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + public ExperimentImportEntity getEntity() { try (ApplicationContext appContext = ApplicationContext.run()) { ProgramLocationService programLocationService = appContext.getBean(ProgramLocationService.class); LocationService locationService = appContext.getBean(LocationService.class); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/WorkflowReadInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/WorkflowReadInitialization.java new file mode 100644 index 000000000..0ca6ed004 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/WorkflowReadInitialization.java @@ -0,0 +1,73 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; + +import io.micronaut.context.annotation.Prototype; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize.WorkflowInitialization; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.List; +import java.util.Optional; + +@Slf4j +@Prototype +public class WorkflowReadInitialization implements BrAPIAction { + private final ExperimentImportEntity entity; // The entity used for read operations initialization + + protected WorkflowReadInitialization(ExperimentImportEntity entity) { + this.entity = entity; + } + + + /** + * Executes the read workflow by fetching members from the entity and initializing the workflow. + * + * @return an Optional containing the BrAPIState representing the completed read workflow + * @throws ApiException if an error occurs during execution + */ + public Optional execute() throws ApiException { + try { + List fetchedMembers = entity.brapiRead(); + entity.initializeWorkflow(fetchedMembers); + return Optional.of(new BrAPIReadWorkflowInitialization.BrAPIReadState(fetchedMembers)); + } catch(ApiException e) { + log.error(String.format("Error fetching %s: %s", entity.getClass().getName(), Utilities.generateApiExceptionLogMessage(e)), e); + throw new ApiException(e); + } + } + + /** + * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. + * + * @return The ExperimentImportEntity representing the BrAPI entity being acted on. + */ + @Override + public ExperimentImportEntity getEntity() { + return null; + } + + /** + * The state class representing the result of a read operation. + * + * @param the type of entity members contained in the state + */ + @Getter + public static class BrAPIReadState implements BrAPIState { + + private final List members; // The list of members fetched during the read operation + + /** + * Constructs a new BrAPIReadState object with the provided list of members. + * + * @param fetchedMembers the list of members fetched during the read operation + */ + public BrAPIReadState(List fetchedMembers) { this.members = fetchedMembers; } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIObservationUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIObservationUpdate.java index 5cf4cec56..4baa3e239 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIObservationUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIObservationUpdate.java @@ -12,13 +12,14 @@ @Slf4j public class BrAPIObservationUpdate extends BrAPIUpdate { - + ExpUnitMiddlewareContext context; public BrAPIObservationUpdate(ExpUnitMiddlewareContext context) { - super(context); + + this.context = context; } @Override - public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { + public ExperimentImportEntity getEntity() { try (ApplicationContext appContext = ApplicationContext.run()) { BrAPIObservationDAO brAPIObservationDAO = appContext.getBean(BrAPIObservationDAO.class); OntologyService ontologyService = appContext.getBean(OntologyService.class); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPITrialUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPITrialUpdate.java index dc9bba419..7792e8165 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPITrialUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPITrialUpdate.java @@ -1,30 +1,28 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update; -import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingTrial; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; @Slf4j public class BrAPITrialUpdate extends BrAPIUpdate { - + ExpUnitMiddlewareContext context; public BrAPITrialUpdate(ExpUnitMiddlewareContext context) { - super(context); + + this.context = context; } @Override - public ExperimentImportEntity getEntity(ExpUnitMiddlewareContext context) { - try (ApplicationContext appContext = ApplicationContext.run()) { - TrialService trialService = appContext.getBean(TrialService.class); - BrAPITrialDAO brAPITrialDAO = appContext.getBean(BrAPITrialDAO.class); - ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); - - return new PendingTrial(context, trialService, brAPITrialDAO, experimentUtilities); - } + public ExperimentImportEntity getEntity() { +// try (ApplicationContext appContext = ApplicationContext.run()) { +// TrialService trialService = appContext.getBean(TrialService.class); +// BrAPITrialDAO brAPITrialDAO = appContext.getBean(BrAPITrialDAO.class); +// ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); +// +// return new PendingTrial(context, trialService, brAPITrialDAO, experimentUtilities); +// } + return pendingEntityFactory.pendingTrialBean(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdate.java index a4928453f..07a9e3482 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdate.java @@ -8,6 +8,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import java.util.List; @@ -16,9 +17,15 @@ @Slf4j public abstract class BrAPIUpdate implements BrAPIAction { ExperimentImportEntity entity; + PendingEntityFactory pendingEntityFactory; - protected BrAPIUpdate(ExpUnitMiddlewareContext context) { - this.entity = getEntity(context); + protected BrAPIUpdate(PendingEntityFactory pendingEntityFactory) { + + this.entity = getEntity(); + this.pendingEntityFactory = pendingEntityFactory; + } + + protected BrAPIUpdate() { } public Optional execute() throws ApiException { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java new file mode 100644 index 000000000..6b873499f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java @@ -0,0 +1,40 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; + +import io.micronaut.context.annotation.Bean; +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; + +import javax.inject.Inject; + +@Factory +public class PendingEntityFactory { + private final TrialService trialService; + private final BrAPITrialDAO brapiTrialDAO; + private final ExperimentUtilities experimentUtilities; + + @Inject + public PendingEntityFactory(TrialService trialService, + BrAPITrialDAO brapiTrialDAO, + ExperimentUtilities experimentUtilities) { + this.trialService = trialService; + this.brapiTrialDAO = brapiTrialDAO; + this.experimentUtilities = experimentUtilities; + } + + public static PendingTrial pendingTrial(ExpUnitMiddlewareContext context, + TrialService trialService, + BrAPITrialDAO brapiTrialDAO, + ExperimentUtilities experimentUtilities) { + return new PendingTrial(context, trialService, brapiTrialDAO, experimentUtilities); + } + + @Bean + @Prototype + public PendingTrial pendingTrialBean(ExpUnitMiddlewareContext context) { + return pendingTrial(context, trialService, brapiTrialDAO, experimentUtilities); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java index 7c8d3ad08..035b4e1be 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; +import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; @@ -17,25 +18,32 @@ import java.util.*; import java.util.stream.Collectors; +@Prototype public class PendingTrial implements ExperimentImportEntity { - ExpUnitContext cache; - ImportContext importContext; - TrialService trialService; - BrAPITrialDAO brapiTrialDAO; - ExperimentUtilities experimentUtilities; + private final ExpUnitContext cache; + private final ImportContext importContext; + private final TrialService trialService; + private final BrAPITrialDAO brapiTrialDAO; + private final ExperimentUtilities experimentUtilities; - @Inject public PendingTrial(ExpUnitMiddlewareContext context, TrialService trialService, - BrAPITrialDAO brAPITrialDAO, + BrAPITrialDAO brapiTrialDAO, ExperimentUtilities experimentUtilities) { this.cache = context.getExpUnitContext(); this.importContext = context.getImportContext(); this.trialService = trialService; - this.brapiTrialDAO = brAPITrialDAO; + this.brapiTrialDAO = brapiTrialDAO; this.experimentUtilities = experimentUtilities; } +// @Inject +// public void setDependencies(TrialService trialService, BrAPITrialDAO brapiTrialDAO, ExperimentUtilities experimentUtilities) { +// this.trialService = trialService; +// this.brapiTrialDAO = brapiTrialDAO; +// this.experimentUtilities = experimentUtilities; +// } + /** * Create new objects generated by the workflow in the BrAPI service. * diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrialProvider.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrialProvider.java new file mode 100644 index 000000000..0b88019b1 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrialProvider.java @@ -0,0 +1,46 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; + +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; + +@Singleton +public class PendingTrialProvider implements Provider { + TrialService trialService; + BrAPITrialDAO brapiTrialDAO; + ExperimentUtilities experimentUtilities; + + @Inject + public PendingTrialProvider(TrialService trialService, + BrAPITrialDAO brAPITrialDAO, + ExperimentUtilities experimentUtilities) { + this.trialService = trialService; + this.brapiTrialDAO = brAPITrialDAO; + this.experimentUtilities = experimentUtilities; + } + + +// public PendingTrial get(ExpUnitMiddlewareContext context) { +// return new PendingTrial(context, trialService, brapiTrialDAO, experimentUtilities); +// } + + /** + * Provides a fully-constructed and injected instance of {@code T}. + * + * @throws RuntimeException if the injector encounters an error while + * providing an instance. For example, if an injectable member on + * {@code T} throws an exception, the injector may wrap the exception + * and throw it to the caller of {@code get()}. Callers should not try + * to handle such exceptions as the behavior may vary across injector + * implementations and even different configurations of the same injector. + */ + @Override + public PendingTrial get() { + return null; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index 3b8140f30..58eb52c56 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -8,10 +8,12 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPITrialCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPITrialUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import javax.inject.Inject; import java.util.Optional; @Slf4j @@ -23,11 +25,16 @@ public class BrAPITrialCommit extends ExpUnitMiddleware { private Optional createdBrAPITrials; private Optional priorBrAPITrials; private Optional updatedTrials; + PendingEntityFactory pendingEntityFactory; + @Inject + public BrAPITrialCommit(PendingEntityFactory pendingEntityFactory) { + this.pendingEntityFactory = pendingEntityFactory; + } @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { - brAPITrialCreation = new BrAPITrialCreation(context); + brAPITrialCreation = new BrAPITrialCreation(context, pendingEntityFactory); createdBrAPITrials = brAPITrialCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); brAPITrialUpdate = new BrAPITrialUpdate(context); priorBrAPITrials = brAPITrialUpdate.getBrAPIState().map(s -> (BrAPIUpdate.BrAPIUpdateState) s); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index cf921f103..60ac592f7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -4,24 +4,34 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.*; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import javax.inject.Inject; + @Slf4j @Prototype public class WorkflowInitialization extends ExpUnitMiddleware { BrAPIObservationUnitReadWorkflowInitialization brAPIObservationUnitReadWorkflowInitialization; - BrAPITrialReadWorkflowInitialization brAPITrialReadWorkflowInitialization; + WorkflowReadInitialization brAPITrialReadWorkflowInitialization; BrAPIStudyReadWorkflowInitialization brAPIStudyReadWorkflowInitialization; LocationReadWorkflowInitialization locationReadWorkflowInitialization; BrAPIDatasetReadWorkflowInitialization brAPIDatasetReadWorkflowInitialization; BrAPIGermplasmReadWorkflowInitialization brAPIGermplasmReadWorkflowInitialization; + PendingEntityFactory pendingEntityFactory; + BrAPIReadFactory brAPIReadFactory; + @Inject + public WorkflowInitialization(PendingEntityFactory pendingEntityFactory, BrAPIReadFactory brAPIReadFactory) { + this.pendingEntityFactory = pendingEntityFactory; + this.brAPIReadFactory = brAPIReadFactory; + } @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { brAPIObservationUnitReadWorkflowInitialization = new BrAPIObservationUnitReadWorkflowInitialization(context); - brAPITrialReadWorkflowInitialization = new BrAPITrialReadWorkflowInitialization(context); + brAPITrialReadWorkflowInitialization = brAPIReadFactory.trialWorkflowReadInitialization(context); brAPIStudyReadWorkflowInitialization = new BrAPIStudyReadWorkflowInitialization(context); locationReadWorkflowInitialization = new LocationReadWorkflowInitialization(context); brAPIDatasetReadWorkflowInitialization = new BrAPIDatasetReadWorkflowInitialization(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredBrAPIData.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredBrAPIData.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredBrAPIData.java index 4009a8454..f4ae66dde 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredBrAPIData.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.misc; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredDatasets.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredDatasets.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredDatasets.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredDatasets.java index f9f773fb2..368086c7a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredDatasets.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredDatasets.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.misc; import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredGermplasm.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredGermplasm.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredGermplasm.java index 096650dae..92b92c04f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredGermplasm.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.misc; import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredLocations.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredLocations.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredLocations.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredLocations.java index e3d8643c4..ae7526a08 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredLocations.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredLocations.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.misc; import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredObservationUnits.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredObservationUnits.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredObservationUnits.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredObservationUnits.java index 99624bd46..101c27c1b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredObservationUnits.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredObservationUnits.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.misc; import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredStudies.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredStudies.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredStudies.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredStudies.java index 59a434e4a..741b625fd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredStudies.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredStudies.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.misc; import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredTrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredTrials.java similarity index 83% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredTrials.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredTrials.java index 374271d0c..1ec85cf6e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/RequiredTrials.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredTrials.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.misc; import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; @@ -29,9 +29,9 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { log.debug("fetching from BrAPI service, trials belonging to required units"); - BrAPITrialReadWorkflowInitialization brAPITrialReadWorkflowInitialization = new BrAPITrialReadWorkflowInitialization(context); - brAPITrialReadWorkflowInitialization.execute(); - + //BrAPITrialReadWorkflowInitialization brAPITrialReadWorkflowInitialization = new BrAPITrialReadWorkflowInitialization(context); + //brAPITrialReadWorkflowInitialization.execute(); + throw new ApiException(); } catch (ApiException e) { this.compensate(context); } diff --git a/src/main/java/org/breedinginsight/utilities/cache/RedissonManager.java b/src/main/java/org/breedinginsight/utilities/cache/RedissonManager.java new file mode 100644 index 000000000..a4dd299a2 --- /dev/null +++ b/src/main/java/org/breedinginsight/utilities/cache/RedissonManager.java @@ -0,0 +1,26 @@ +package org.breedinginsight.utilities.cache; + +import io.micronaut.context.annotation.Value; +import org.redisson.Redisson; +import org.redisson.api.RedissonClient; +import org.redisson.config.Config; +import io.micronaut.context.event.ApplicationEventListener; +import io.micronaut.runtime.server.event.ServerShutdownEvent; + +import javax.inject.Singleton; +import java.io.IOException; + +@Singleton +public class RedissonManager implements ApplicationEventListener { + private final RedissonClient redissonClient; + + public RedissonManager(@Value("${redisson.single-server-config}") String configFile) throws IOException { + Config config = Config.fromYAML(RedissonManager.class.getResourceAsStream(configFile)); + this.redissonClient = Redisson.create(config); + } + + @Override + public void onApplicationEvent(ServerShutdownEvent event) { + redissonClient.shutdown(); + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 10acaf1f0..c207e0bee 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -182,6 +182,7 @@ email: from: ${EMAIL_FROM} redisson: + shutdown-on-stop: false single-server-config: address: ${REDIS_URL:`redis://localhost:6379`} connect-timeout: ${REDIS_TIMEOUT:30s} From 669751e7c385c523fb9bb4edb26e44c5ec0bc7f2 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 13 Jun 2024 22:20:22 -0400 Subject: [PATCH 124/203] add obs unit to read factory --- .../read/BrAPIDatasetReadWorkflowInitialization.java | 4 ++-- .../BrAPIGermplasmReadWorkflowInitialization.java | 2 +- ...APIObservationUnitReadWorkflowInitialization.java | 2 +- .../action/read/BrAPIReadFactory.java | 12 ++++++++++++ .../read/BrAPIStudyReadWorkflowInitialization.java | 2 +- .../read/LocationReadWorkflowInitialization.java | 4 ++-- .../action/read/WorkflowReadInitialization.java | 5 +---- .../{ => misc}/BrAPIReadWorkflowInitialization.java | 2 +- .../action/read/{ => misc}/BrAPITrialRead.java | 3 ++- .../BrAPITrialReadWorkflowInitialization.java | 2 +- .../initialize/WorkflowInitialization.java | 9 ++++++--- .../middleware/misc/RequiredTrials.java | 1 - 12 files changed, 30 insertions(+), 18 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/{ => misc}/BrAPIReadWorkflowInitialization.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/{ => misc}/BrAPITrialRead.java (88%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/{ => misc}/BrAPITrialReadWorkflowInitialization.java (96%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java index ca51d1a46..5e22534f1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java @@ -4,13 +4,13 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIReadWorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingDataset; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; -public class BrAPIDatasetReadWorkflowInitialization extends BrAPIReadWorkflowInitialization{ +public class BrAPIDatasetReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { /** * Constructs a new BrAPIReadWorkflowInitialization object with the given ExpUnitMiddlewareContext. * Initializes the entity based on the provided context. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java index 351ef620d..a233bb669 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java @@ -3,8 +3,8 @@ import io.micronaut.context.ApplicationContext; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIReadWorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingGermplasm; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.GermplasmService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java index f124edd6e..12e8ebf35 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java @@ -4,8 +4,8 @@ import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIReadWorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java index 447969a92..b4cac1e64 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java @@ -4,6 +4,7 @@ import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Prototype; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; @@ -23,9 +24,20 @@ public static WorkflowReadInitialization trialWorkflowReadInitializa return new WorkflowReadInitialization(pendingEntityFactory.pendingTrialBean(context)); } + public static WorkflowReadInitialization observationUnitWorkflowReadInitialization(ExpUnitMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowReadInitialization(pendingEntityFactory.pendingTrialBean(context)); + } + @Bean @Prototype public WorkflowReadInitialization trialWorkflowReadInitialization(ExpUnitMiddlewareContext context) { return trialWorkflowReadInitialization(context, pendingEntityFactory); } + + @Bean + @Prototype + public WorkflowReadInitialization observationUnitWorkflowReadInitialization(ExpUnitMiddlewareContext context) { + return observationUnitWorkflowReadInitialization(context, pendingEntityFactory); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java index 6f48b0ce1..8ee5da48d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java @@ -4,8 +4,8 @@ import org.brapi.v2.model.core.BrAPIStudy; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIReadWorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingStudy; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java index fbac2e0fd..7aa3a5bf4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java @@ -3,8 +3,8 @@ import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIReadWorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingLocation; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; @@ -12,7 +12,7 @@ import org.breedinginsight.services.ProgramLocationService; @Slf4j -public class LocationReadWorkflowInitialization extends BrAPIReadWorkflowInitialization{ +public class LocationReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { ExpUnitMiddlewareContext context; /** * Constructs a new BrAPIReadWorkflowInitialization object with the given ExpUnitMiddlewareContext. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/WorkflowReadInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/WorkflowReadInitialization.java index 0ca6ed004..7310bd1b5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/WorkflowReadInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/WorkflowReadInitialization.java @@ -6,13 +6,10 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIReadWorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize.WorkflowInitialization; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.utilities.Utilities; -import javax.inject.Inject; import java.util.List; import java.util.Optional; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIReadWorkflowInitialization.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIReadWorkflowInitialization.java index 04f00b824..f19e8ceb3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIReadWorkflowInitialization.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.Getter; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialRead.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialRead.java similarity index 88% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialRead.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialRead.java index 628c78b68..157557606 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialRead.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialRead.java @@ -1,7 +1,8 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.BrAPIRead; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialReadWorkflowInitialization.java similarity index 96% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialReadWorkflowInitialization.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialReadWorkflowInitialization.java index 342c727f8..a36ab85d2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPITrialReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialReadWorkflowInitialization.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.core.BrAPITrial; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index 60ac592f7..15d8d0b97 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -3,7 +3,10 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.*; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.BrAPIStudyReadWorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; @@ -14,8 +17,8 @@ @Slf4j @Prototype public class WorkflowInitialization extends ExpUnitMiddleware { - BrAPIObservationUnitReadWorkflowInitialization brAPIObservationUnitReadWorkflowInitialization; - WorkflowReadInitialization brAPITrialReadWorkflowInitialization; + WorkflowReadInitialization brAPIObservationUnitReadWorkflowInitialization; + WorkflowReadInitialization brAPITrialReadWorkflowInitialization; BrAPIStudyReadWorkflowInitialization brAPIStudyReadWorkflowInitialization; LocationReadWorkflowInitialization locationReadWorkflowInitialization; BrAPIDatasetReadWorkflowInitialization brAPIDatasetReadWorkflowInitialization; @@ -30,7 +33,7 @@ public WorkflowInitialization(PendingEntityFactory pendingEntityFactory, BrAPIRe } @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { - brAPIObservationUnitReadWorkflowInitialization = new BrAPIObservationUnitReadWorkflowInitialization(context); + brAPIObservationUnitReadWorkflowInitialization = brAPIReadFactory.observationUnitWorkflowReadInitialization(context); brAPITrialReadWorkflowInitialization = brAPIReadFactory.trialWorkflowReadInitialization(context); brAPIStudyReadWorkflowInitialization = new BrAPIStudyReadWorkflowInitialization(context); locationReadWorkflowInitialization = new LocationReadWorkflowInitialization(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredTrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredTrials.java index 1ec85cf6e..a7da97d12 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredTrials.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredTrials.java @@ -3,7 +3,6 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.BrAPITrialReadWorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; From 5bc9e1a151da1f44b1346d76a3da1d9291965b31 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 13 Jun 2024 22:55:07 -0400 Subject: [PATCH 125/203] add obs unit to entity factory --- .../action/read/BrAPIReadFactory.java | 2 +- ...rvationUnitReadWorkflowInitialization.java | 2 +- .../entity/PendingEntityFactory.java | 21 +++++++++++++++++++ .../entity/PendingObservationUnit.java | 2 +- .../appendoverwrite/entity/PendingTrial.java | 7 ------- 5 files changed, 24 insertions(+), 10 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/{ => misc}/BrAPIObservationUnitReadWorkflowInitialization.java (98%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java index b4cac1e64..373e99f47 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java @@ -26,7 +26,7 @@ public static WorkflowReadInitialization trialWorkflowReadInitializa public static WorkflowReadInitialization observationUnitWorkflowReadInitialization(ExpUnitMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { - return new WorkflowReadInitialization(pendingEntityFactory.pendingTrialBean(context)); + return new WorkflowReadInitialization(pendingEntityFactory.pendingObservationUnitBean(context)); } @Bean diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIObservationUnitReadWorkflowInitialization.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIObservationUnitReadWorkflowInitialization.java index 12e8ebf35..7e6c7a21a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIObservationUnitReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIObservationUnitReadWorkflowInitialization.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; import io.micronaut.context.ApplicationContext; import org.brapi.v2.model.pheno.BrAPIObservationUnit; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java index 6b873499f..84c1ac970 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java @@ -3,9 +3,11 @@ import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; import javax.inject.Inject; @@ -14,14 +16,20 @@ public class PendingEntityFactory { private final TrialService trialService; private final BrAPITrialDAO brapiTrialDAO; + private final BrAPIObservationUnitDAO observationUnitDAO; + private final ObservationUnitService observationUnitService; private final ExperimentUtilities experimentUtilities; @Inject public PendingEntityFactory(TrialService trialService, BrAPITrialDAO brapiTrialDAO, + BrAPIObservationUnitDAO observationUnitDAO, + ObservationUnitService observationUnitService, ExperimentUtilities experimentUtilities) { this.trialService = trialService; this.brapiTrialDAO = brapiTrialDAO; + this.observationUnitDAO = observationUnitDAO; + this.observationUnitService = observationUnitService; this.experimentUtilities = experimentUtilities; } @@ -32,9 +40,22 @@ public static PendingTrial pendingTrial(ExpUnitMiddlewareContext context, return new PendingTrial(context, trialService, brapiTrialDAO, experimentUtilities); } + public static PendingObservationUnit pendingObservationUnit(ExpUnitMiddlewareContext context, + BrAPIObservationUnitDAO observationUnitDAO, + ObservationUnitService observationUnitService, + ExperimentUtilities experimentUtilities) { + return new PendingObservationUnit(context, observationUnitDAO, observationUnitService, experimentUtilities); + } + @Bean @Prototype public PendingTrial pendingTrialBean(ExpUnitMiddlewareContext context) { return pendingTrial(context, trialService, brapiTrialDAO, experimentUtilities); } + + @Bean + @Prototype + public PendingObservationUnit pendingObservationUnitBean(ExpUnitMiddlewareContext context) { + return pendingObservationUnit(context, observationUnitDAO, observationUnitService, experimentUtilities); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java index 41f404629..ea6efea3e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java @@ -29,7 +29,7 @@ public class PendingObservationUnit implements ExperimentImportEntity Date: Fri, 14 Jun 2024 14:24:47 -0400 Subject: [PATCH 126/203] Building --- .../ExperimentImportService.java | 2 + .../model/workflow/ImportContext.java | 2 +- .../model/workflow/ProcessedData.java | 8 +- .../AppendOverwritePhenotypesWorkflow.java | 3 +- .../workflow/CreateNewExperimentWorkflow.java | 106 ++++++++++++-- .../steps/CommitPendingImportObjectsStep.java | 3 +- ...ulateExistingPendingImportObjectsStep.java | 3 +- .../PopulateNewPendingImportObjectsStep.java | 134 ++++++++---------- .../ValidatePendingImportObjectsStep.java | 35 +++-- .../CreateNewEnvironmentWorkflow.java | 3 +- 10 files changed, 192 insertions(+), 107 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index fededeea1..616b6fa58 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -34,6 +34,7 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import java.util.HashMap; import java.util.List; @Singleton @@ -85,6 +86,7 @@ public ImportPreviewResponse process(ImportServiceContext context) .data(context.getData()) .commit(context.isCommit()) .upload(context.getUpload()) + .program(context.getProgram()) .build(); return context.getWorkflow().process(importContext); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java index 109c300cc..f9205cb87 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java @@ -40,7 +40,7 @@ public class ImportContext { private ImportUpload upload; private List importRows; // TODO: move this out potentially - private Map mappedBrAPIImport; + //private Map mappedBrAPIImport; private Table data; private Program program; private User user; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java index 73302bdb9..677f35469 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -18,19 +18,19 @@ package org.breedinginsight.brapps.importer.model.workflow; import lombok.*; -import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; -import java.util.ArrayList; -import java.util.List; import java.util.Map; @Data @ToString @NoArgsConstructor public class ProcessedData { - private Map statistics; + // TODO: remove this, already in ImportPreviewResponse + //private Map statistics; + // TODO private Map mappedBrAPIImport; + private ImportPreviewResponse importPreviewResponse; } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index 21df19be6..124a3a0f9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -18,6 +18,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; import org.breedinginsight.brapps.importer.model.workflow.Workflow; @@ -33,7 +34,7 @@ @Named("AppendOverwritePhenotypesWorkflow") public class AppendOverwritePhenotypesWorkflow implements Workflow { @Override - public ProcessedData process(ImportContext context) { + public ImportPreviewResponse process(ImportContext context) { // TODO return null; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index e32a5c29c..343b4475d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -23,17 +23,22 @@ import lombok.extern.slf4j.Slf4j; import lombok.val; import org.apache.commons.lang3.StringUtils; +import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.ImportStatusService; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.CommitPendingImportObjectsStep; @@ -46,6 +51,7 @@ import javax.inject.Inject; import javax.inject.Named; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -86,11 +92,13 @@ public ImportPreviewResponse process(ImportContext context) throws Exception { ImportUpload upload = context.getUpload(); boolean commit = context.isCommit(); + List importRows = context.getImportRows(); // Make sure the file does not contain obs unit ids before proceeding if (containsObsUnitIDs(context)) { // TODO: get file name - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Error detected in file, XXX.xls. ObsUnitIDs are detected. Import cannot proceed"); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Error detected in file, " + + upload.getUploadFileName() + ". ObsUnitIDs are detected. Import cannot proceed"); } statusService.updateMessage(upload, "Checking existing experiment objects in brapi service and mapping data"); @@ -98,14 +106,15 @@ public ImportPreviewResponse process(ImportContext context) throws Exception { ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(context); ProcessContext processContext = populateExistingPendingImportObjectsStep.process(context, phenotypeData); ProcessedData processedData = populateNewPendingImportObjectsStep.process(processContext, phenotypeData); - ValidationErrors validationErrors = validatePendingImportObjectsStep.process(context); + ValidationErrors validationErrors = validatePendingImportObjectsStep.process(context, processContext.getPendingData(), phenotypeData, processedData); // short circuit if there were validation errors if (validationErrors.hasErrors()) { throw new ValidatorException(validationErrors); } - ImportPreviewResponse response = buildImportPreviewResponse(processedData, upload); + // TODO: move to experiment import service + ImportPreviewResponse response = buildImportPreviewResponse(importRows, processContext.getPendingData(), processedData, upload); statusService.updateMappedData(upload, response, "Finished mapping data to brapi objects"); @@ -116,7 +125,7 @@ public ImportPreviewResponse process(ImportContext context) throws Exception { } // commit data - long totalObjects = getNewObjectCount(processedData); + long totalObjects = getNewObjectCount(response); statusService.startUpload(upload, totalObjects, "Starting upload to brapi service"); statusService.updateMessage(upload, "Creating new experiment objects in brapi service"); @@ -138,10 +147,11 @@ public String getName() { } // TODO: move to shared area - private ImportPreviewResponse buildImportPreviewResponse(ProcessedData processedData, + private ImportPreviewResponse buildImportPreviewResponse(List importRows, PendingData pendingData, ProcessedData processedData, ImportUpload upload) { - Map statistics = processedData.getStatistics(); + Map mappedBrAPIImport = processedData.getMappedBrAPIImport(); + Map statistics = generateStatisticsMap(pendingData, importRows); ImportPreviewResponse response = new ImportPreviewResponse(); response.setStatistics(statistics); @@ -152,10 +162,10 @@ private ImportPreviewResponse buildImportPreviewResponse(ProcessedData processed } // TODO: move to shared area - private long getNewObjectCount(ProcessedData processedData) { + private long getNewObjectCount(ImportPreviewResponse response) { // get total number of new brapi objects to create long totalObjects = 0; - for (ImportPreviewStatistics stats : processedData.getStatistics().values()) { + for (ImportPreviewStatistics stats : response.getStatistics().values()) { totalObjects += stats.getNewObjectCount(); } return totalObjects; @@ -169,5 +179,85 @@ private boolean containsObsUnitIDs(ImportContext importContext) { return StringUtils.isNotBlank(expRow.getObsUnitID()); }); } + + // TODO: move to shared area: experiment import service + private Map generateStatisticsMap(PendingData pendingData, List importRows) { + // Data for stats. + HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names + HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's + HashSet gidCounter = new HashSet<>(); // set of unique GID's + + Map> observationByHash = pendingData.getObservationByHash(); + + for (BrAPIImport row : importRows) { + ExperimentObservation importRow = (ExperimentObservation) row; + // Collect date for stats. + addIfNotNull(environmentNameCounter, importRow.getEnv()); + addIfNotNull(obsUnitsIDCounter, ExperimentUtilities.createObservationUnitKey(importRow)); + addIfNotNull(gidCounter, importRow.getGid()); + } + + int numNewObservations = Math.toIntExact( + observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + int numExistingObservations = Math.toIntExact( + observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.EXISTING && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + int numMutatedObservations = Math.toIntExact( + observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() + .newObjectCount(environmentNameCounter.size()) + .build(); + ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() + .newObjectCount(obsUnitsIDCounter.size()) + .build(); + ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() + .newObjectCount(gidCounter.size()) + .build(); + ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder() + .newObjectCount(numNewObservations) + .build(); + ImportPreviewStatistics existingObservationStats = ImportPreviewStatistics.builder() + .newObjectCount(numExistingObservations) + .build(); + ImportPreviewStatistics mutatedObservationStats = ImportPreviewStatistics.builder() + .newObjectCount(numMutatedObservations) + .build(); + + return Map.of( + "Environments", environmentStats, + "Observation_Units", obdUnitStats, + "GIDs", gidStats, + "Observations", observationStats, + "Existing_Observations", existingObservationStats, + "Mutated_Observations", mutatedObservationStats + ); + } + + // TODO: move to common area + private void addIfNotNull(HashSet set, String setValue) { + if (setValue != null) { + set.add(setValue); + } + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index fa02c76ed..12e6bcc99 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -95,6 +95,7 @@ public void process(ProcessContext processContext, ProcessedData processedData) ImportUpload upload = importContext.getUpload(); Program program = importContext.getProgram(); + Map mappedBrAPIImport = processedData.getMappedBrAPIImport(); Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); @@ -159,7 +160,7 @@ public void process(ProcessContext processContext, ProcessedData processedData) .setLocationDbId(createdLocation.getLocationDbId()); } - updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); + updateStudyDependencyValues(pendingData, mappedBrAPIImport, program.getKey()); List createdStudies = brAPIStudyDAO.createBrAPIStudies(newStudies, program.getId(), upload); // set the DbId to the for each newly created study diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index 22799a025..c5196e662 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -120,6 +120,7 @@ public ProcessContext process(ImportContext input, ProcessedPhenotypeData phenot .obsVarDatasetByName(obsVarDatasetByName) .existingGermplasmByGID(existingGermplasmByGID) .existingObsByObsHash(existingObsByObsHash) + .observationByHash(new HashMap<>()) .build(); return ProcessContext.builder() @@ -539,7 +540,7 @@ private Map fetchExistingObservations(List refe String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); - String key = ExperimentUtilities.getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); + String key = ExperimentUtilities.getObservationHash(ExperimentUtilities.createObservationUnitKey(studyName, ouName), variableName, studyName); return Map.entry(key, obs); }) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index 2d22895fc..a471b84f3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -16,12 +16,13 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; +import com.google.gson.Gson; import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpStatus; import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.apache.xmlbeans.impl.xb.xsdschema.ImportDocument; +import org.brapi.client.v2.JSON; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; @@ -77,6 +78,7 @@ public class PopulateNewPendingImportObjectsStep { private final BrAPIObservationDAO brAPIObservationDAO; private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final DSLContext dsl; + private final Gson gson; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @@ -92,6 +94,7 @@ public PopulateNewPendingImportObjectsStep(ExperimentValidateService experimentV this.brAPIObservationDAO = brAPIObservationDAO; this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.dsl = dsl; + this.gson = new JSON().getGson(); } public ProcessedData process(ProcessContext context, ProcessedPhenotypeData phenotypeData) @@ -131,13 +134,11 @@ private void populatePendingImportObjects(ProcessContext processContext, for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - populateIndependentVariablePIOsForRow(importContext, phenotypeData, pendingData, importRow, expNextVal, envNextVal); + PendingImportObjectData pioData = populateIndependentVariablePIOsForRow(importContext, + phenotypeData, pendingData, importRow, expNextVal, envNextVal); - // ... (Common logic from the original method) - - PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); - - processObservations(importContext, phenotypeData, importRow, rowNum, commit, importRow.getEnvYear(), obsUnitPIO, studyPIO); + processObservations(importContext, + phenotypeData, pendingData, importRow, rowNum, commit, pioData.getObsUnitPIO(), pioData.getStudyPIO()); } } @@ -247,10 +248,14 @@ private PendingImportObjectData populateIndependentVariablePIOsForRow(ImportCont .build(); } - private void processObservations(ImportContext importContext, ProcessedPhenotypeData phenotypeData, ExperimentObservation importRow, - int rowNum, boolean commit, String studyYear, - PendingImportObject obsUnitPIO, - PendingImportObject studyPIO) + // TODO: some reusable stuff between workflows that could potentially be broken out + private void processObservations(ImportContext importContext, + ProcessedPhenotypeData phenotypeData, + PendingData pendingData, + ExperimentObservation importRow, + int rowNum, boolean commit, + PendingImportObject obsUnitPIO, + PendingImportObject studyPIO) throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { Program program = importContext.getProgram(); User user = importContext.getUser(); @@ -259,56 +264,35 @@ private void processObservations(ImportContext importContext, ProcessedPhenotype List referencedTraits = phenotypeData.getReferencedTraits(); for (Column column : phenotypeCols) { - // ... (Logic for processing observations) - } - } - - // TODO: move common code out - private void initNewBrapiData(ImportContext importContext, ProcessedPhenotypeData phenotypeData) - throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { - - Program program = importContext.getProgram(); - List importRows = importContext.getImportRows(); - User user = importContext.getUser(); - boolean commit = importContext.isCommit(); - Map existingObsByObsHash; - - List referencedTraits = phenotypeData.getReferencedTraits(); - List> phenotypeCols = phenotypeData.getPhenotypeCols(); - Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); - - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - - - for (Column column : phenotypeCols) { - //If associated timestamp column, add - String dateTimeValue = null; - if (timeStampColByPheno.containsKey(column.name())) { - dateTimeValue = timeStampColByPheno.get(column.name()).getString(rowNum); - //If no timestamp, set to midnight - if (!dateTimeValue.isBlank() && !validDateTimeValue(dateTimeValue)) { - dateTimeValue += MIDNIGHT; - } + //If associated timestamp column, add + String dateTimeValue = null; + if (timeStampColByPheno.containsKey(column.name())) { + dateTimeValue = timeStampColByPheno.get(column.name()).getString(rowNum); + //If no timestamp, set to midnight + if (!dateTimeValue.isBlank() && !validDateTimeValue(dateTimeValue)) { + dateTimeValue += MIDNIGHT; } - - // get the study year either referenced from the observation unit or listed explicitly on the import row - // TODO: handle this different workflows - String studyYear = hasAllReferenceUnitIds ? studyPIO.getBrAPIObject().getSeasons().get(0) : importRow.getEnvYear(); - String seasonDbId = experimentSeasonService.yearToSeasonDbId(studyYear, program.getId()); - fetchOrCreateObservationPIO( - program, - user, - importRow, - column, //column.name() gets phenotype name - rowNum, - dateTimeValue, - commit, - seasonDbId, - obsUnitPIO, - studyPIO, - referencedTraits - ); } + + // get the study year either referenced from the observation unit or listed explicitly on the import row + // NOTE: removed append / update code + String studyYear = importRow.getEnvYear(); + String seasonDbId = experimentSeasonService.yearToSeasonDbId(studyYear, program.getId()); + fetchOrCreateObservationPIO( + phenotypeData, + pendingData, + program, + user, + importRow, + column, //column.name() gets phenotype name + rowNum, + dateTimeValue, + commit, + seasonDbId, + obsUnitPIO, + studyPIO, + referencedTraits + ); } } @@ -563,7 +547,9 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Import return pio; } - private void fetchOrCreateObservationPIO(Program program, + private void fetchOrCreateObservationPIO(ProcessedPhenotypeData phenotypeData, + PendingData pendingData, + Program program, User user, ExperimentObservation importRow, Column column, @@ -579,24 +565,21 @@ private void fetchOrCreateObservationPIO(Program program, String variableName = column.name(); String value = column.getString(rowNum); String key; + Map existingObsByObsHash = pendingData.getExistingObsByObsHash(); + Map> observationByHash = pendingData.getObservationByHash(); + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); - // TODO: multiple workflows - if (hasAllReferenceUnitIds) { - String unitName = obsUnitPIO.getBrAPIObject().getObservationUnitName(); - String studyName = studyPIO.getBrAPIObject().getStudyName(); - key = getObservationHash(studyName + unitName, variableName, studyName); - } else { - key = getImportObservationHash(importRow, variableName); - } + // NOTE: removed append / update + key = getImportObservationHash(importRow, variableName); if (existingObsByObsHash.containsKey(key)) { - if (!isObservationMatched(key, value, column, rowNum)){ + if (!isObservationMatched(phenotypeData, pendingData, key, value, column, rowNum)){ // prior observation with updated value newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); - if (!isValueMatched(key, value)){ + if (!isValueMatched(pendingData, key, value)){ newObservation.setValue(value); - } else if (!StringUtils.isBlank(timeStampValue) && !isTimestampMatched(key, timeStampValue)) { + } else if (!StringUtils.isBlank(timeStampValue) && !isTimestampMatched(pendingData, key, timeStampValue)) { DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timeStampValue)); newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); @@ -609,12 +592,11 @@ private void fetchOrCreateObservationPIO(Program program, } observationByHash.put(key, pio); - } else if (!this.observationByHash.containsKey(key)){ + } else if (!observationByHash.containsKey(key)){ // new observation - // TODO: multiple workflows - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + // NOTE: removed append / update code + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); UUID studyID = studyPIO.getId(); @@ -629,7 +611,7 @@ private void fetchOrCreateObservationPIO(Program program, newObservation.setStudyDbId(studyPIO.getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); - this.observationByHash.put(key, pio); + observationByHash.put(key, pio); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java index 4ccdcd868..99fbc12eb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java @@ -39,6 +39,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; @@ -54,10 +55,7 @@ import javax.inject.Singleton; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; @Singleton @Slf4j @@ -78,18 +76,27 @@ public ValidatePendingImportObjectsStep(ExperimentSeasonService experimentSeason this.gson = new JSON().getGson(); } - public ValidationErrors process(ImportContext input) { - ValidationErrors validationErrors = new ValidationErrors(); + public ValidationErrors process(ImportContext importContext, PendingData pendingData, ProcessedPhenotypeData phenotypeData, ProcessedData processedData) { + //Map mappedBrAPIImport = processedData.getMappedBrAPIImport(); + List importRows = importContext.getImportRows(); + List> phenotypeCols = phenotypeData.getPhenotypeCols(); + Program program = importContext.getProgram(); + List referencedTraits = phenotypeData.getReferencedTraits(); + boolean commit = importContext.isCommit(); + User user = importContext.getUser(); + Map mappedBrAPIImport = prepareDataForValidation(importRows, pendingData, phenotypeCols); + ValidationErrors validationErrors = validateFields(importRows, mappedBrAPIImport, referencedTraits, program, phenotypeCols, commit, user, pendingData, phenotypeData); + processedData.setMappedBrAPIImport(mappedBrAPIImport); return validationErrors; - } - private void prepareDataForValidation(List importRows, + private Map prepareDataForValidation(List importRows, PendingData pendingData, - List> phenotypeCols, - Map mappedBrAPIImport) { + List> phenotypeCols) { + + Map mappedBrAPIImport = new HashMap<>(); Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); @@ -97,7 +104,7 @@ private void prepareDataForValidation(List importRows, Map> locationByName = pendingData.getLocationByName(); Map> obsVarDatasetByName = pendingData.getObsVarDatasetByName(); Map> existingGermplasmByGID = pendingData.getExistingGermplasmByGID(); - + Map> observationByHash = pendingData.getObservationByHash(); for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); @@ -121,6 +128,8 @@ private void prepareDataForValidation(List importRows, mappedBrAPIImport.put(rowNum, mappedImportRow); } + + return mappedBrAPIImport; } private PendingImportObject getGidPIO(PendingData pendingData, ExperimentObservation importRow) { @@ -368,6 +377,7 @@ private void validateObservations(PendingData pendingData, Map existingObsByObsHash = pendingData.getExistingObsByObsHash(); Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); + Map> observationByHash = pendingData.getObservationByHash(); phenotypeCols.forEach(phenoCol -> { String importHash; @@ -458,7 +468,4 @@ private void validateObservations(PendingData pendingData, } }); } - - - } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java index 5776ab59b..05269964f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java @@ -18,6 +18,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; import org.breedinginsight.brapps.importer.model.workflow.Workflow; @@ -33,7 +34,7 @@ @Named("CreateNewEnvironmentWorkflow") public class CreateNewEnvironmentWorkflow implements Workflow { @Override - public ProcessedData process(ImportContext context) { + public ImportPreviewResponse process(ImportContext context) { // TODO return null; } From d57e4aced1386de7d3d3fdafb7d54043e686358d Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:29:06 -0400 Subject: [PATCH 127/203] Renamed to have github actions run --- ...riment_workflows.sql => V1.23.0__add_experiment_workflows.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V1.22.0__add_experiment_workflows.sql => V1.23.0__add_experiment_workflows.sql} (100%) diff --git a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql similarity index 100% rename from src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql rename to src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql From 787f9194b1428648657931e81b9da4440c9c78e6 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 14 Jun 2024 16:19:50 -0400 Subject: [PATCH 128/203] add remaining entitties to entity factory --- .../entity/PendingDataset.java | 4 +- .../entity/PendingEntityFactory.java | 97 ++++++++++++++++++- .../entity/PendingGermplasm.java | 3 +- .../entity/PendingLocation.java | 4 +- .../entity/PendingObservation.java | 3 +- .../appendoverwrite/entity/PendingStudy.java | 3 +- .../entity/PendingTrialProvider.java | 46 --------- 7 files changed, 105 insertions(+), 55 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrialProvider.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java index 4c7770b90..1a588d186 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; +import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.request.BrAPIListNewRequest; @@ -23,13 +24,14 @@ import java.util.Map; import java.util.stream.Collectors; +@Prototype public class PendingDataset implements ExperimentImportEntity { ExpUnitContext cache; ImportContext importContext; BrAPIListDAO brAPIListDAO; DatasetService datasetService; ExperimentUtilities experimentUtilities; - @Inject + public PendingDataset(ExpUnitMiddlewareContext context, BrAPIListDAO brAPIListDAO, DatasetService datasetService, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java index 84c1ac970..77e785279 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java @@ -3,12 +3,12 @@ import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Prototype; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; -import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.*; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.ProgramLocationService; import javax.inject.Inject; @@ -18,6 +18,15 @@ public class PendingEntityFactory { private final BrAPITrialDAO brapiTrialDAO; private final BrAPIObservationUnitDAO observationUnitDAO; private final ObservationUnitService observationUnitService; + private final StudyService studyService; + private final BrAPIStudyDAO brAPIStudyDAO; + private final GermplasmService germplasmService; + private final BrAPIListDAO brAPIListDAO; + private final DatasetService datasetService; + private final BrAPIObservationDAO brAPIObservationDAO; + private final OntologyService ontologyService; + private final ProgramLocationService programLocationService; + private final LocationService locationService; private final ExperimentUtilities experimentUtilities; @Inject @@ -25,11 +34,27 @@ public PendingEntityFactory(TrialService trialService, BrAPITrialDAO brapiTrialDAO, BrAPIObservationUnitDAO observationUnitDAO, ObservationUnitService observationUnitService, + StudyService studyService, + BrAPIStudyDAO brAPIStudyDAO, + GermplasmService germplasmService, + BrAPIListDAO brAPIListDAO, + DatasetService datasetService, + BrAPIObservationDAO brAPIObservationDAO, + OntologyService ontologyService, ProgramLocationService programLocationService, LocationService locationService, ExperimentUtilities experimentUtilities) { this.trialService = trialService; this.brapiTrialDAO = brapiTrialDAO; this.observationUnitDAO = observationUnitDAO; this.observationUnitService = observationUnitService; + this.studyService = studyService; + this.brAPIStudyDAO = brAPIStudyDAO; + this.germplasmService = germplasmService; + this.brAPIListDAO = brAPIListDAO; + this.datasetService = datasetService; + this.brAPIObservationDAO = brAPIObservationDAO; + this.ontologyService = ontologyService; + this.programLocationService = programLocationService; + this.locationService = locationService; this.experimentUtilities = experimentUtilities; } @@ -47,6 +72,40 @@ public static PendingObservationUnit pendingObservationUnit(ExpUnitMiddlewareCon return new PendingObservationUnit(context, observationUnitDAO, observationUnitService, experimentUtilities); } + public static PendingStudy pendingStudy(ExpUnitMiddlewareContext context, + StudyService studyService, + BrAPIStudyDAO brAPIStudyDAO, + ExperimentUtilities experimentUtilities) { + return new PendingStudy(context, studyService, brAPIStudyDAO, experimentUtilities); + } + + public static PendingGermplasm pendingGermplasm(ExpUnitMiddlewareContext context, + GermplasmService germplasmService, + ExperimentUtilities experimentUtilities) { + return new PendingGermplasm(context, germplasmService, experimentUtilities); + } + + public static PendingDataset pendingDataset(ExpUnitMiddlewareContext context, + BrAPIListDAO brAPIListDAO, + DatasetService datasetService, + ExperimentUtilities experimentUtilities) { + return new PendingDataset(context, brAPIListDAO, datasetService, experimentUtilities); + } + + public static PendingObservation pendingObservation(ExpUnitMiddlewareContext context, + BrAPIObservationDAO brAPIObservationDAO, + OntologyService ontologyService, + ExperimentUtilities experimentUtilities) { + return new PendingObservation(context, brAPIObservationDAO, ontologyService, experimentUtilities); + } + + public static PendingLocation pendingLocation(ExpUnitMiddlewareContext context, + ProgramLocationService programLocationService, + LocationService locationService, + ExperimentUtilities experimentUtilities) { + return new PendingLocation(context, programLocationService, locationService, experimentUtilities); + } + @Bean @Prototype public PendingTrial pendingTrialBean(ExpUnitMiddlewareContext context) { @@ -58,4 +117,34 @@ public PendingTrial pendingTrialBean(ExpUnitMiddlewareContext context) { public PendingObservationUnit pendingObservationUnitBean(ExpUnitMiddlewareContext context) { return pendingObservationUnit(context, observationUnitDAO, observationUnitService, experimentUtilities); } + + @Bean + @Prototype + public PendingStudy pendingStudyBean(ExpUnitMiddlewareContext context) { + return pendingStudy(context, studyService, brAPIStudyDAO, experimentUtilities); + } + + @Bean + @Prototype + public PendingGermplasm pendingGermplasmBean(ExpUnitMiddlewareContext context) { + return pendingGermplasm(context, germplasmService, experimentUtilities); + } + + @Bean + @Prototype + public PendingDataset pendingDataset(ExpUnitMiddlewareContext context) { + return pendingDataset(context, brAPIListDAO, datasetService, experimentUtilities); + } + + @Bean + @Prototype + public PendingObservation pendingObservation(ExpUnitMiddlewareContext context) { + return pendingObservation(context, brAPIObservationDAO, ontologyService, experimentUtilities); + } + + @Bean + @Prototype + public PendingLocation pendingLocation(ExpUnitMiddlewareContext context) { + return pendingLocation(context, programLocationService, locationService, experimentUtilities); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java index 624f48b57..8dc8b0e6f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; +import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -17,13 +18,13 @@ import java.util.*; import java.util.stream.Collectors; +@Prototype public class PendingGermplasm implements ExperimentImportEntity { ExpUnitContext cache; ImportContext importContext; GermplasmService germplasmService; ExperimentUtilities experimentUtilities; - @Inject public PendingGermplasm(ExpUnitMiddlewareContext context, GermplasmService germplasmService, ExperimentUtilities experimentUtilities) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java index 03607bd1b..64c40bef6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; +import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; @@ -23,13 +24,14 @@ import java.util.Set; import java.util.stream.Collectors; +@Prototype public class PendingLocation implements ExperimentImportEntity { ExpUnitContext cache; ImportContext importContext; ProgramLocationService programLocationService; LocationService locationService; ExperimentUtilities experimentUtilities; - @Inject + public PendingLocation(ExpUnitMiddlewareContext context, ProgramLocationService programLocationService, LocationService locationService, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java index 1d55c43d2..686e507e7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; +import io.micronaut.context.annotation.Prototype; import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservation; @@ -20,6 +21,7 @@ import java.util.Optional; import java.util.stream.Collectors; +@Prototype public class PendingObservation implements ExperimentImportEntity { ExpUnitContext cache; ImportContext importContext; @@ -27,7 +29,6 @@ public class PendingObservation implements ExperimentImportEntity{ ExpUnitContext cache; ImportContext importContext; @@ -29,7 +31,6 @@ public class PendingStudy implements ExperimentImportEntity{ BrAPIStudyDAO brAPIStudyDAO; ExperimentUtilities experimentUtilities; - @Inject public PendingStudy(ExpUnitMiddlewareContext context, StudyService studyService, BrAPIStudyDAO brAPIStudyDAO, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrialProvider.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrialProvider.java deleted file mode 100644 index 0b88019b1..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrialProvider.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; - -import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; - -import javax.inject.Inject; -import javax.inject.Provider; -import javax.inject.Singleton; - -@Singleton -public class PendingTrialProvider implements Provider { - TrialService trialService; - BrAPITrialDAO brapiTrialDAO; - ExperimentUtilities experimentUtilities; - - @Inject - public PendingTrialProvider(TrialService trialService, - BrAPITrialDAO brAPITrialDAO, - ExperimentUtilities experimentUtilities) { - this.trialService = trialService; - this.brapiTrialDAO = brAPITrialDAO; - this.experimentUtilities = experimentUtilities; - } - - -// public PendingTrial get(ExpUnitMiddlewareContext context) { -// return new PendingTrial(context, trialService, brapiTrialDAO, experimentUtilities); -// } - - /** - * Provides a fully-constructed and injected instance of {@code T}. - * - * @throws RuntimeException if the injector encounters an error while - * providing an instance. For example, if an injectable member on - * {@code T} throws an exception, the injector may wrap the exception - * and throw it to the caller of {@code get()}. Callers should not try - * to handle such exceptions as the behavior may vary across injector - * implementations and even different configurations of the same injector. - */ - @Override - public PendingTrial get() { - return null; - } -} From e466598ad9785bc9284207138ea8cae852d8e30e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 14 Jun 2024 17:32:09 -0400 Subject: [PATCH 129/203] fill out read factory --- .../action/read/BrAPIReadFactory.java | 54 +++++++++++++++++-- ...rAPIDatasetReadWorkflowInitialization.java | 2 +- ...PIGermplasmReadWorkflowInitialization.java | 2 +- .../action/read/{ => misc}/BrAPIRead.java | 2 +- .../BrAPIStudyReadWorkflowInitialization.java | 2 +- .../action/read/misc/BrAPITrialRead.java | 2 - .../LocationReadWorkflowInitialization.java | 2 +- .../entity/PendingEntityFactory.java | 6 +-- .../initialize/WorkflowInitialization.java | 28 ++++++---- 9 files changed, 76 insertions(+), 24 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/{ => misc}/BrAPIDatasetReadWorkflowInitialization.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/{ => misc}/BrAPIGermplasmReadWorkflowInitialization.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/{ => misc}/BrAPIRead.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/{ => misc}/BrAPIStudyReadWorkflowInitialization.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/{ => misc}/LocationReadWorkflowInitialization.java (98%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java index 373e99f47..f12816708 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java @@ -3,10 +3,14 @@ import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Prototype; +import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.model.ProgramLocation; import javax.inject.Inject; @@ -25,19 +29,63 @@ public static WorkflowReadInitialization trialWorkflowReadInitializa } public static WorkflowReadInitialization observationUnitWorkflowReadInitialization(ExpUnitMiddlewareContext context, - PendingEntityFactory pendingEntityFactory) { + PendingEntityFactory pendingEntityFactory) { return new WorkflowReadInitialization(pendingEntityFactory.pendingObservationUnitBean(context)); } + public static WorkflowReadInitialization germplasmWorkflowReadInitialization(ExpUnitMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowReadInitialization(pendingEntityFactory.pendingGermplasmBean(context)); + } + + public static WorkflowReadInitialization datasetWorkflowReadInitialization(ExpUnitMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowReadInitialization(pendingEntityFactory.pendingDatasetBean(context)); + } + + public static WorkflowReadInitialization studyWorkflowReadInitialization(ExpUnitMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowReadInitialization(pendingEntityFactory.pendingStudyBean(context)); + } + + public static WorkflowReadInitialization locationWorkflowReadInitialization(ExpUnitMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowReadInitialization(pendingEntityFactory.pendingLocationBean(context)); + } + @Bean @Prototype - public WorkflowReadInitialization trialWorkflowReadInitialization(ExpUnitMiddlewareContext context) { + public WorkflowReadInitialization trialWorkflowReadInitializationBean(ExpUnitMiddlewareContext context) { return trialWorkflowReadInitialization(context, pendingEntityFactory); } @Bean @Prototype - public WorkflowReadInitialization observationUnitWorkflowReadInitialization(ExpUnitMiddlewareContext context) { + public WorkflowReadInitialization observationUnitWorkflowReadInitializationBean(ExpUnitMiddlewareContext context) { return observationUnitWorkflowReadInitialization(context, pendingEntityFactory); } + + @Bean + @Prototype + public WorkflowReadInitialization germplasmWorkflowReadInitializationBean(ExpUnitMiddlewareContext context) { + return germplasmWorkflowReadInitialization(context, pendingEntityFactory); + } + + @Bean + @Prototype + public WorkflowReadInitialization datasetWorkflowReadInitializationBean(ExpUnitMiddlewareContext context) { + return datasetWorkflowReadInitialization(context, pendingEntityFactory); + } + + @Bean + @Prototype + public WorkflowReadInitialization studyWorkflowReadInitializationBean(ExpUnitMiddlewareContext context) { + return studyWorkflowReadInitialization(context, pendingEntityFactory); + } + + @Bean + @Prototype + public WorkflowReadInitialization locationWorkflowReadInitializationBean(ExpUnitMiddlewareContext context) { + return locationWorkflowReadInitialization(context, pendingEntityFactory); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIDatasetReadWorkflowInitialization.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIDatasetReadWorkflowInitialization.java index 5e22534f1..6740135d2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIDatasetReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIDatasetReadWorkflowInitialization.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; import io.micronaut.context.ApplicationContext; import org.brapi.v2.model.core.response.BrAPIListDetails; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIGermplasmReadWorkflowInitialization.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIGermplasmReadWorkflowInitialization.java index a233bb669..afa840b53 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIGermplasmReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIGermplasmReadWorkflowInitialization.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; import io.micronaut.context.ApplicationContext; import org.brapi.v2.model.germ.BrAPIGermplasm; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIRead.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIRead.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIRead.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIRead.java index f3901743c..82201dea4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIRead.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIRead.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.Getter; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIStudyReadWorkflowInitialization.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIStudyReadWorkflowInitialization.java index 8ee5da48d..beb01b3de 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIStudyReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIStudyReadWorkflowInitialization.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; import io.micronaut.context.ApplicationContext; import org.brapi.v2.model.core.BrAPIStudy; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialRead.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialRead.java index 157557606..af136d465 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialRead.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialRead.java @@ -2,9 +2,7 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.BrAPIRead; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; @Slf4j diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/LocationReadWorkflowInitialization.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/LocationReadWorkflowInitialization.java index 7aa3a5bf4..0d8285703 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/LocationReadWorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/LocationReadWorkflowInitialization.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java index 77e785279..b73b12415 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java @@ -132,19 +132,19 @@ public PendingGermplasm pendingGermplasmBean(ExpUnitMiddlewareContext context) { @Bean @Prototype - public PendingDataset pendingDataset(ExpUnitMiddlewareContext context) { + public PendingDataset pendingDatasetBean(ExpUnitMiddlewareContext context) { return pendingDataset(context, brAPIListDAO, datasetService, experimentUtilities); } @Bean @Prototype - public PendingObservation pendingObservation(ExpUnitMiddlewareContext context) { + public PendingObservation pendingObservationBean(ExpUnitMiddlewareContext context) { return pendingObservation(context, brAPIObservationDAO, ontologyService, experimentUtilities); } @Bean @Prototype - public PendingLocation pendingLocation(ExpUnitMiddlewareContext context) { + public PendingLocation pendingLocationBean(ExpUnitMiddlewareContext context) { return pendingLocation(context, programLocationService, locationService, experimentUtilities); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index 15d8d0b97..97c587d09 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -3,14 +3,20 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.*; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.BrAPIStudyReadWorkflowInitialization; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIStudyReadWorkflowInitialization; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIDatasetReadWorkflowInitialization; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIGermplasmReadWorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.model.ProgramLocation; import javax.inject.Inject; @@ -19,10 +25,10 @@ public class WorkflowInitialization extends ExpUnitMiddleware { WorkflowReadInitialization brAPIObservationUnitReadWorkflowInitialization; WorkflowReadInitialization brAPITrialReadWorkflowInitialization; - BrAPIStudyReadWorkflowInitialization brAPIStudyReadWorkflowInitialization; - LocationReadWorkflowInitialization locationReadWorkflowInitialization; - BrAPIDatasetReadWorkflowInitialization brAPIDatasetReadWorkflowInitialization; - BrAPIGermplasmReadWorkflowInitialization brAPIGermplasmReadWorkflowInitialization; + WorkflowReadInitialization brAPIStudyReadWorkflowInitialization; + WorkflowReadInitialization locationReadWorkflowInitialization; + WorkflowReadInitialization brAPIDatasetReadWorkflowInitialization; + WorkflowReadInitialization brAPIGermplasmReadWorkflowInitialization; PendingEntityFactory pendingEntityFactory; BrAPIReadFactory brAPIReadFactory; @@ -33,12 +39,12 @@ public WorkflowInitialization(PendingEntityFactory pendingEntityFactory, BrAPIRe } @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { - brAPIObservationUnitReadWorkflowInitialization = brAPIReadFactory.observationUnitWorkflowReadInitialization(context); - brAPITrialReadWorkflowInitialization = brAPIReadFactory.trialWorkflowReadInitialization(context); - brAPIStudyReadWorkflowInitialization = new BrAPIStudyReadWorkflowInitialization(context); - locationReadWorkflowInitialization = new LocationReadWorkflowInitialization(context); - brAPIDatasetReadWorkflowInitialization = new BrAPIDatasetReadWorkflowInitialization(context); - brAPIGermplasmReadWorkflowInitialization = new BrAPIGermplasmReadWorkflowInitialization(context); + brAPIObservationUnitReadWorkflowInitialization = brAPIReadFactory.observationUnitWorkflowReadInitializationBean(context); + brAPITrialReadWorkflowInitialization = brAPIReadFactory.trialWorkflowReadInitializationBean(context); + brAPIStudyReadWorkflowInitialization = brAPIReadFactory.studyWorkflowReadInitializationBean(context); + locationReadWorkflowInitialization = brAPIReadFactory.locationWorkflowReadInitializationBean(context); + brAPIDatasetReadWorkflowInitialization = brAPIReadFactory.datasetWorkflowReadInitializationBean(context); + brAPIGermplasmReadWorkflowInitialization = brAPIReadFactory.germplasmWorkflowReadInitializationBean(context); log.debug("reading required BrAPI data from BrAPI service"); try { From afeb1c1a6f471ecab5aa5b940fa8ad8b56d1782f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 14 Jun 2024 18:49:59 -0400 Subject: [PATCH 130/203] create brapi creation factory --- .../action/create/BrAPICreationFactory.java | 91 +++++++++++++++++++ .../action/create/WorkflowCreation.java | 91 +++++++++++++++++++ .../create/{ => misc}/BrAPICreation.java | 2 +- .../{ => misc}/BrAPIDatasetCreation.java | 2 +- .../{ => misc}/BrAPIObservationCreation.java | 2 +- .../BrAPIObservationUnitCreation.java | 2 +- .../create/{ => misc}/BrAPIStudyCreation.java | 2 +- .../create/{ => misc}/BrAPITrialCreation.java | 2 +- .../create/{ => misc}/LocationCreation.java | 4 +- .../middleware/commit/BrAPIDatasetCommit.java | 4 +- .../commit/BrAPIObservationCommit.java | 4 +- .../commit/BrAPIObservationUnitCommit.java | 4 +- .../middleware/commit/BrAPIStudyCommit.java | 4 +- .../middleware/commit/BrAPITrialCommit.java | 4 +- .../middleware/commit/LocationCommit.java | 4 +- .../{ => misc}/BrAPIDatasetCreation.java | 2 +- 16 files changed, 203 insertions(+), 21 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreationFactory.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/WorkflowCreation.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/{ => misc}/BrAPICreation.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/{ => misc}/BrAPIDatasetCreation.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/{ => misc}/BrAPIObservationCreation.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/{ => misc}/BrAPIObservationUnitCreation.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/{ => misc}/BrAPIStudyCreation.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/{ => misc}/BrAPITrialCreation.java (96%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/{ => misc}/LocationCreation.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/{ => misc}/BrAPIDatasetCreation.java (98%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreationFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreationFactory.java new file mode 100644 index 000000000..9099607f8 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreationFactory.java @@ -0,0 +1,91 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; + +import io.micronaut.context.annotation.Bean; +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Prototype; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.model.ProgramLocation; + +import javax.inject.Inject; + +@Factory +public class BrAPICreationFactory { + private final PendingEntityFactory pendingEntityFactory; + + @Inject + public BrAPICreationFactory(PendingEntityFactory pendingEntityFactory) { + this.pendingEntityFactory = pendingEntityFactory; + } + + public static WorkflowCreation trialWorkflowCreation(ExpUnitMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowCreation(pendingEntityFactory.pendingTrialBean(context)); + } + + public static WorkflowCreation datasetWorkflowCreation(ExpUnitMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowCreation(pendingEntityFactory.pendingDatasetBean(context)); + } + + public static WorkflowCreation studyWorkflowCreation(ExpUnitMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowCreation(pendingEntityFactory.pendingStudyBean(context)); + } + + public static WorkflowCreation observationWorkflowCreation(ExpUnitMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowCreation(pendingEntityFactory.pendingObservationBean(context)); + } + + public static WorkflowCreation observationUnitWorkflowCreation(ExpUnitMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowCreation(pendingEntityFactory.pendingObservationUnitBean(context)); + } + + public static WorkflowCreation locationWorkflowCreation(ExpUnitMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowCreation(pendingEntityFactory.pendingLocationBean(context)); + } + + @Bean + @Prototype + public WorkflowCreation trialWorkflowCreationBean(ExpUnitMiddlewareContext context) { + return trialWorkflowCreation(context, pendingEntityFactory); + } + + @Bean + @Prototype + public WorkflowCreation datasetWorkflowCreationBean(ExpUnitMiddlewareContext context) { + return datasetWorkflowCreation(context, pendingEntityFactory); + } + + @Bean + @Prototype + public WorkflowCreation studyWorkflowCreationBean(ExpUnitMiddlewareContext context) { + return studyWorkflowCreation(context, pendingEntityFactory); + } + + @Bean + @Prototype + public WorkflowCreation observationWorkflowCreationBean(ExpUnitMiddlewareContext context) { + return observationWorkflowCreation(context, pendingEntityFactory); + } + + @Bean + @Prototype + public WorkflowCreation observationUnitWorkflowCreationBean(ExpUnitMiddlewareContext context) { + return observationUnitWorkflowCreation(context, pendingEntityFactory); + } + + @Bean + @Prototype + public WorkflowCreation locationWorkflowCreationBean(ExpUnitMiddlewareContext context) { + return locationWorkflowCreation(context, pendingEntityFactory); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/WorkflowCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/WorkflowCreation.java new file mode 100644 index 000000000..1a7e8e810 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/WorkflowCreation.java @@ -0,0 +1,91 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; + +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import java.util.List; +import java.util.Optional; + +@Slf4j +@Prototype +public class WorkflowCreation implements BrAPIAction { + + private final ExperimentImportEntity entity; + + + protected WorkflowCreation(ExperimentImportEntity entity) { + this.entity = entity; + } + + + + /** + * Executes the creation process for entities. + * @return an Optional containing the BrAPI state after execution + * @throws ApiException if an error occurs during execution + */ + public Optional execute() throws ApiException { + List newMembers = entity.copyWorkflowMembers(ImportObjectState.NEW); + try { + List createdMembers = entity.brapiPost(newMembers); + entity.updateWorkflow(createdMembers); + return Optional.of(new BrAPICreationState<>(createdMembers)); + } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { + log.error("Error creating..."); + throw new InternalServerException("Error creating...", e); + } + } + + /** + * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. + * + * @return The ExperimentImportEntity representing the BrAPI entity being acted on. + */ + @Override + public ExperimentImportEntity getEntity() { + return null; + } + + /** + * Inner class representing the state of creation for BrAPI entities. + * @param the type of entity + */ + @Getter + public class BrAPICreationState implements BrAPIState { + + private final List members; + + /** + * Constructor for BrAPICreationState class. + * @param createdMembers the list of created members + */ + public BrAPICreationState(List createdMembers) { + this.members = createdMembers; + } + + /** + * Undo the creation operation by deleting the created members. + * @return true if undo operation is successful, false otherwise + */ + public boolean undo() { + List createdMembers = this.getMembers(); + try { + return entity.brapiDelete(createdMembers); + } catch (ApiException e) { + log.error("Error deleting..."); + throw new InternalServerException("Error deleting...", e); + } + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPICreation.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPICreation.java index 2def125e1..5a497e993 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPICreation.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.Getter; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIDatasetCreation.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIDatasetCreation.java index b22f7c447..227650c14 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIDatasetCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIDatasetCreation.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc; import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIObservationCreation.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationCreation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIObservationCreation.java index d034d09a4..45cb24254 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIObservationCreation.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc; import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationUnitCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIObservationUnitCreation.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationUnitCreation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIObservationUnitCreation.java index 08ab6311c..3f284040c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIObservationUnitCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIObservationUnitCreation.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc; import io.micronaut.context.ApplicationContext; import org.brapi.v2.model.pheno.BrAPIObservationUnit; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIStudyCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIStudyCreation.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIStudyCreation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIStudyCreation.java index 6995b5731..69062693c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPIStudyCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIStudyCreation.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc; import io.micronaut.context.ApplicationContext; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPITrialCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPITrialCreation.java similarity index 96% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPITrialCreation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPITrialCreation.java index b6bba6e7b..721bdd3cb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPITrialCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPITrialCreation.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.core.BrAPITrial; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/LocationCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/LocationCreation.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/LocationCreation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/LocationCreation.java index 9a18bbef5..4744ac7c1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/LocationCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/LocationCreation.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc; import io.micronaut.context.ApplicationContext; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; @@ -9,7 +9,7 @@ import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.services.ProgramLocationService; -public class LocationCreation extends BrAPICreation{ +public class LocationCreation extends BrAPICreation { ExpUnitMiddlewareContext context; /** * Constructor for BrAPICreation class. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index 388d701d6..0e71f3e85 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -2,8 +2,8 @@ import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPIDatasetCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPIDatasetCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java index c76464ba1..ae52820b6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -4,8 +4,8 @@ import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPIObservationCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPIObservationCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIObservationUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index 7e54e9b14..e4cde1d07 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -2,8 +2,8 @@ import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPIObservationUnitCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPIObservationUnitCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java index 2ff3fa1b5..2595ceaaa 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -4,8 +4,8 @@ import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPIStudyCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPIStudyCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index 58eb52c56..bee83da8d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -4,8 +4,8 @@ import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPITrialCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPITrialCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPITrialUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index ab5cbf8b4..ba31aef49 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -2,8 +2,8 @@ import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.LocationCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.LocationCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPIDatasetCreation.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCreation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPIDatasetCreation.java index 77b1a9126..ca66ccdce 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPIDatasetCreation.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.misc; import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; From 37d6ff35f4c46b462b8c6c81c9ffd325957023af Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 14 Jun 2024 19:31:26 -0400 Subject: [PATCH 131/203] inject creation factory into commit middleware --- .../middleware/commit/BrAPIDatasetCommit.java | 14 ++++++++++++-- .../commit/BrAPIObservationCommit.java | 14 +++++++++++--- .../commit/BrAPIObservationUnitCommit.java | 18 +++++++++++------- .../middleware/commit/BrAPIStudyCommit.java | 14 +++++++++++--- .../middleware/commit/BrAPITrialCommit.java | 14 ++++++++------ .../middleware/commit/LocationCommit.java | 14 +++++++++++--- .../BrAPILocationCreationMiddleware.java | 2 +- .../BrAPIObservationUpdateMiddleware.java | 2 +- .../{ => misc}/BrAPITrialUpdateMiddleware.java | 2 +- .../initialize/WorkflowInitialization.java | 4 +--- 10 files changed, 68 insertions(+), 30 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/{ => misc}/BrAPILocationCreationMiddleware.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/{ => misc}/BrAPIObservationUpdateMiddleware.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/{ => misc}/BrAPITrialUpdateMiddleware.java (97%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index 0e71f3e85..ea6fdc737 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -2,23 +2,33 @@ import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPIDatasetCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import javax.inject.Inject; import java.util.Optional; @Prototype public class BrAPIDatasetCommit extends ExpUnitMiddleware { - private BrAPIDatasetCreation datasetCreation; + private BrAPICreationFactory brAPICreationFactory; + private WorkflowCreation datasetCreation; private Optional createdDatasets; + + @Inject + public BrAPIDatasetCommit(BrAPICreationFactory brAPICreationFactory) { + this.brAPICreationFactory = brAPICreationFactory; + } @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { - datasetCreation = new BrAPIDatasetCreation(context); + datasetCreation = brAPICreationFactory.datasetWorkflowCreationBean(context); createdDatasets = datasetCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); } catch (ApiException e) { context.getExpUnitContext().setProcessError(new MiddlewareError(e)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java index ae52820b6..7376ac157 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -4,6 +4,9 @@ import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPIObservationCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIObservationUpdate; @@ -12,22 +15,27 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import javax.inject.Inject; import java.util.Optional; @Slf4j @Prototype -@NoArgsConstructor public class BrAPIObservationCommit extends ExpUnitMiddleware { - private BrAPIObservationCreation brAPIObservationCreation; + private BrAPICreationFactory brAPICreationFactory; + private WorkflowCreation brAPIObservationCreation; private BrAPIObservationUpdate brAPIObservationUpdate; private Optional createdBrAPIObservations; private Optional priorBrAPIObservations; private Optional updatedObservations; + @Inject + public BrAPIObservationCommit(BrAPICreationFactory brAPICreationFactory) { + this.brAPICreationFactory = brAPICreationFactory; + } @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { - brAPIObservationCreation = new BrAPIObservationCreation(context); + brAPIObservationCreation = brAPICreationFactory.observationWorkflowCreationBean(context); createdBrAPIObservations = brAPIObservationCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); brAPIObservationUpdate = new BrAPIObservationUpdate(context); priorBrAPIObservations = brAPIObservationUpdate.getBrAPIState().map(s -> (BrAPIUpdate.BrAPIUpdateState) s); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index e4cde1d07..7b5deb55c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -2,28 +2,32 @@ import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPIObservationUnitCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import javax.inject.Inject; import java.util.Optional; @Prototype public class BrAPIObservationUnitCommit extends ExpUnitMiddleware { - private BrAPIObservationUnitCreation brAPIObservationUnitCreation; + private BrAPICreationFactory brAPICreationFactory; + private WorkflowCreation brAPIObservationUnitCreation; private Optional createdBrAPIObservationUnits; - /** - * Subclasses will implement this local transaction. - * - * @param context - */ + @Inject + public BrAPIObservationUnitCommit(BrAPICreationFactory brAPICreationFactory) { + this.brAPICreationFactory = brAPICreationFactory; + } @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try{ - brAPIObservationUnitCreation = new BrAPIObservationUnitCreation(context); + brAPIObservationUnitCreation = brAPICreationFactory.observationUnitWorkflowCreationBean(context); createdBrAPIObservationUnits = brAPIObservationUnitCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); } catch (ApiException e) { context.getExpUnitContext().setProcessError(new MiddlewareError(e)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java index 2595ceaaa..b779e23be 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -4,25 +4,33 @@ import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPIStudyCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import javax.inject.Inject; import java.util.Optional; @Slf4j @Prototype -@NoArgsConstructor public class BrAPIStudyCommit extends ExpUnitMiddleware { - private BrAPIStudyCreation brAPIStudyCreation; + private BrAPICreationFactory brAPICreationFactory; + private WorkflowCreation brAPIStudyCreation; private Optional createdBrAPIStudies; + @Inject + public BrAPIStudyCommit(BrAPICreationFactory brAPICreationFactory) { + this.brAPICreationFactory = brAPICreationFactory; + } @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { - brAPIStudyCreation = new BrAPIStudyCreation(context); + brAPIStudyCreation = brAPICreationFactory.studyWorkflowCreationBean(context); createdBrAPIStudies = brAPIStudyCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); } catch (ApiException e) { context.getExpUnitContext().setProcessError(new MiddlewareError(e)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index bee83da8d..23547c0c2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -4,6 +4,9 @@ import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPITrialCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPITrialUpdate; @@ -18,23 +21,22 @@ @Slf4j @Prototype -@NoArgsConstructor public class BrAPITrialCommit extends ExpUnitMiddleware { - private BrAPITrialCreation brAPITrialCreation; + private BrAPICreationFactory brAPICreationFactory; + private WorkflowCreation brAPITrialCreation; private BrAPITrialUpdate brAPITrialUpdate; private Optional createdBrAPITrials; private Optional priorBrAPITrials; private Optional updatedTrials; - PendingEntityFactory pendingEntityFactory; @Inject - public BrAPITrialCommit(PendingEntityFactory pendingEntityFactory) { - this.pendingEntityFactory = pendingEntityFactory; + public BrAPITrialCommit(BrAPICreationFactory brAPICreationFactory) { + this.brAPICreationFactory = brAPICreationFactory; } @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { - brAPITrialCreation = new BrAPITrialCreation(context, pendingEntityFactory); + brAPITrialCreation = brAPICreationFactory.trialWorkflowCreationBean(context); createdBrAPITrials = brAPITrialCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); brAPITrialUpdate = new BrAPITrialUpdate(context); priorBrAPITrials = brAPITrialUpdate.getBrAPIState().map(s -> (BrAPIUpdate.BrAPIUpdateState) s); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index ba31aef49..48a2a6ed6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -2,23 +2,31 @@ import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.LocationCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.model.ProgramLocation; +import javax.inject.Inject; import java.util.Optional; @Prototype public class LocationCommit extends ExpUnitMiddleware { - private LocationCreation locationCreation; + private BrAPICreationFactory brAPICreationFactory; + private WorkflowCreation locationCreation; private Optional createdLocations; + @Inject + public LocationCommit(BrAPICreationFactory brAPICreationFactory) { + this.brAPICreationFactory = brAPICreationFactory; + } @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { - locationCreation = new LocationCreation(context); + locationCreation = brAPICreationFactory.locationWorkflowCreationBean(context); createdLocations = locationCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); } catch (ApiException e) { context.getExpUnitContext().setProcessError(new MiddlewareError(e)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreationMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPILocationCreationMiddleware.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreationMiddleware.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPILocationCreationMiddleware.java index 1e7c900d6..882635d8e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPILocationCreationMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPILocationCreationMiddleware.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.misc; import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUpdateMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPIObservationUpdateMiddleware.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUpdateMiddleware.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPIObservationUpdateMiddleware.java index 3c8461223..3463af873 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUpdateMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPIObservationUpdateMiddleware.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.misc; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.server.exceptions.InternalServerException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdateMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPITrialUpdateMiddleware.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdateMiddleware.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPITrialUpdateMiddleware.java index d02771211..cdea7f417 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialUpdateMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPITrialUpdateMiddleware.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.misc; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.server.exceptions.InternalServerException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index 97c587d09..b6e1b03df 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -29,12 +29,10 @@ public class WorkflowInitialization extends ExpUnitMiddleware { WorkflowReadInitialization locationReadWorkflowInitialization; WorkflowReadInitialization brAPIDatasetReadWorkflowInitialization; WorkflowReadInitialization brAPIGermplasmReadWorkflowInitialization; - PendingEntityFactory pendingEntityFactory; BrAPIReadFactory brAPIReadFactory; @Inject - public WorkflowInitialization(PendingEntityFactory pendingEntityFactory, BrAPIReadFactory brAPIReadFactory) { - this.pendingEntityFactory = pendingEntityFactory; + public WorkflowInitialization(BrAPIReadFactory brAPIReadFactory) { this.brAPIReadFactory = brAPIReadFactory; } @Override From 7daee23498676386f414a6b08e99081ccf72bd9b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 14 Jun 2024 20:27:38 -0400 Subject: [PATCH 132/203] create update factory --- .../action/update/BrAPIObservationUpdate.java | 31 -------- .../action/update/BrAPITrialUpdate.java | 28 ------- .../action/update/BrAPIUpdateFactory.java | 43 ++++++++++ .../action/update/WorkflowUpdate.java | 78 +++++++++++++++++++ .../action/update/{ => misc}/BrAPIUpdate.java | 2 +- .../middleware/commit/BrAPIDatasetCommit.java | 6 +- .../commit/BrAPIObservationCommit.java | 33 ++++---- .../commit/BrAPIObservationUnitCommit.java | 6 +- .../middleware/commit/BrAPIStudyCommit.java | 6 +- .../middleware/commit/BrAPITrialCommit.java | 34 ++++---- .../middleware/commit/LocationCommit.java | 6 +- 11 files changed, 166 insertions(+), 107 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIObservationUpdate.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPITrialUpdate.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdateFactory.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/WorkflowUpdate.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/{ => misc}/BrAPIUpdate.java (98%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIObservationUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIObservationUpdate.java deleted file mode 100644 index 4baa3e239..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIObservationUpdate.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update; - -import io.micronaut.context.ApplicationContext; -import lombok.extern.slf4j.Slf4j; -import org.brapi.v2.model.pheno.BrAPIObservation; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingObservation; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.services.OntologyService; - -@Slf4j -public class BrAPIObservationUpdate extends BrAPIUpdate { - ExpUnitMiddlewareContext context; - public BrAPIObservationUpdate(ExpUnitMiddlewareContext context) { - - this.context = context; - } - - @Override - public ExperimentImportEntity getEntity() { - try (ApplicationContext appContext = ApplicationContext.run()) { - BrAPIObservationDAO brAPIObservationDAO = appContext.getBean(BrAPIObservationDAO.class); - OntologyService ontologyService = appContext.getBean(OntologyService.class); - ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); - - return new PendingObservation(context, brAPIObservationDAO, ontologyService, experimentUtilities); - } - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPITrialUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPITrialUpdate.java deleted file mode 100644 index 7792e8165..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPITrialUpdate.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update; - -import lombok.extern.slf4j.Slf4j; -import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; - -@Slf4j -public class BrAPITrialUpdate extends BrAPIUpdate { - ExpUnitMiddlewareContext context; - public BrAPITrialUpdate(ExpUnitMiddlewareContext context) { - - this.context = context; - } - - @Override - public ExperimentImportEntity getEntity() { -// try (ApplicationContext appContext = ApplicationContext.run()) { -// TrialService trialService = appContext.getBean(TrialService.class); -// BrAPITrialDAO brAPITrialDAO = appContext.getBean(BrAPITrialDAO.class); -// ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); -// -// return new PendingTrial(context, trialService, brAPITrialDAO, experimentUtilities); -// } - return pendingEntityFactory.pendingTrialBean(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdateFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdateFactory.java new file mode 100644 index 000000000..af3291620 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdateFactory.java @@ -0,0 +1,43 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update; + +import io.micronaut.context.annotation.Bean; +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Prototype; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +import javax.inject.Inject; + +@Factory +public class BrAPIUpdateFactory { + private final PendingEntityFactory pendingEntityFactory; + + @Inject + public BrAPIUpdateFactory(PendingEntityFactory pendingEntityFactory) { + this.pendingEntityFactory = pendingEntityFactory; + } + + public static WorkflowUpdate trialWorkflowUpdate(ExpUnitMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowUpdate(pendingEntityFactory.pendingTrialBean(context)); + } + + public static WorkflowUpdate observationWorkflowUpdate(ExpUnitMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowUpdate(pendingEntityFactory.pendingObservationBean(context)); + } + + @Bean + @Prototype + public WorkflowUpdate trialWorkflowUpdateBean(ExpUnitMiddlewareContext context) { + return trialWorkflowUpdate(context, pendingEntityFactory); + } + + @Bean + @Prototype + public WorkflowUpdate observationWorkflowUpdateBean(ExpUnitMiddlewareContext context) { + return observationWorkflowUpdate(context, pendingEntityFactory); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/WorkflowUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/WorkflowUpdate.java new file mode 100644 index 000000000..e57a45ddb --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/WorkflowUpdate.java @@ -0,0 +1,78 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update; + +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; + +import java.util.List; +import java.util.Optional; + +@Slf4j +@Prototype +public class WorkflowUpdate implements BrAPIAction { + private final ExperimentImportEntity entity; + + protected WorkflowUpdate(ExperimentImportEntity entity) { + + this.entity = entity; + } + + public Optional execute() throws ApiException { + return saveAndUpdateCache(entity.copyWorkflowMembers(ImportObjectState.MUTATED)); + } + + /** + * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. + * + * @return The ExperimentImportEntity representing the BrAPI entity being acted on. + */ + @Override + public ExperimentImportEntity getEntity() { + return null; + } + + public Optional> getBrAPIState() { + try { + return Optional.of(new BrAPIUpdateState(entity.getBrAPIState(ImportObjectState.MUTATED))); + } catch (ApiException e) { + // TODO: add specific error messages to entity service + log.error("Error getting..."); + throw new InternalServerException("Error getting...", e); + } + + } + + protected Optional saveAndUpdateCache(List members) { + return Optional.ofNullable(members).map(changes -> { + try { + List savedMembers = entity.brapiPut(changes); + // TODO: set updated fields of workflow brapi object + // entity.updateWorkflow(savedMembers); + return new BrAPIUpdateState(savedMembers); + } catch (ApiException e) { + // TODO: add specific error messages to entity service + log.error("Error updating..."); + throw new InternalServerException("Error updating...", e); + } + }); + } + + @Getter + public class BrAPIUpdateState implements BrAPIState { + private final List members; + + public BrAPIUpdateState(List existingMembers) { this.members = existingMembers; } + + public boolean restore() { + return saveAndUpdateCache(this.getMembers()).isPresent(); + } + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/misc/BrAPIUpdate.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdate.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/misc/BrAPIUpdate.java index 07a9e3482..c7e2c534a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/misc/BrAPIUpdate.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.misc; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.Getter; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index ea6fdc737..8f6eb46f8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -18,7 +18,7 @@ public class BrAPIDatasetCommit extends ExpUnitMiddleware { private BrAPICreationFactory brAPICreationFactory; private WorkflowCreation datasetCreation; - private Optional createdDatasets; + private Optional createdDatasets; @Inject public BrAPIDatasetCommit(BrAPICreationFactory brAPICreationFactory) { @@ -29,7 +29,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { datasetCreation = brAPICreationFactory.datasetWorkflowCreationBean(context); - createdDatasets = datasetCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); + createdDatasets = datasetCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); } catch (ApiException e) { context.getExpUnitContext().setProcessError(new MiddlewareError(e)); return this.compensate(context); @@ -43,7 +43,7 @@ public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); // Delete any created datasets - createdDatasets.ifPresent(BrAPICreation.BrAPICreationState::undo); + createdDatasets.ifPresent(WorkflowCreation.BrAPICreationState::undo); // Undo the prior local transaction return compensatePrior(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java index 7376ac157..c4d33a79f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -1,16 +1,13 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; import io.micronaut.context.annotation.Prototype; -import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPIObservationCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIObservationUpdate; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIUpdateFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; @@ -21,25 +18,27 @@ @Slf4j @Prototype public class BrAPIObservationCommit extends ExpUnitMiddleware { - private BrAPICreationFactory brAPICreationFactory; + private final BrAPICreationFactory brAPICreationFactory; + private final BrAPIUpdateFactory brAPIUpdateFactory; private WorkflowCreation brAPIObservationCreation; - private BrAPIObservationUpdate brAPIObservationUpdate; - private Optional createdBrAPIObservations; - private Optional priorBrAPIObservations; - private Optional updatedObservations; + private WorkflowUpdate brAPIObservationUpdate; + private Optional createdBrAPIObservations; + private Optional priorBrAPIObservations; + private Optional updatedObservations; @Inject - public BrAPIObservationCommit(BrAPICreationFactory brAPICreationFactory) { + public BrAPIObservationCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUpdateFactory brAPIUpdateFactory) { this.brAPICreationFactory = brAPICreationFactory; + this.brAPIUpdateFactory = brAPIUpdateFactory; } @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { brAPIObservationCreation = brAPICreationFactory.observationWorkflowCreationBean(context); - createdBrAPIObservations = brAPIObservationCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); - brAPIObservationUpdate = new BrAPIObservationUpdate(context); - priorBrAPIObservations = brAPIObservationUpdate.getBrAPIState().map(s -> (BrAPIUpdate.BrAPIUpdateState) s); - updatedObservations = brAPIObservationUpdate.execute().map(s -> (BrAPIUpdate.BrAPIUpdateState) s); + createdBrAPIObservations = brAPIObservationCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); + brAPIObservationUpdate = brAPIUpdateFactory.observationWorkflowUpdateBean(context); + priorBrAPIObservations = brAPIObservationUpdate.getBrAPIState().map(s -> s); + updatedObservations = brAPIObservationUpdate.execute().map(s -> (WorkflowUpdate.BrAPIUpdateState) s); } catch (ApiException e) { context.getExpUnitContext().setProcessError(new MiddlewareError(e)); @@ -55,10 +54,10 @@ public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); // Delete any created observations from the BrAPI service - createdBrAPIObservations.ifPresent(BrAPICreation.BrAPICreationState::undo); + createdBrAPIObservations.ifPresent(WorkflowCreation.BrAPICreationState::undo); // Revert any changes made to observations in the BrAPI service - priorBrAPIObservations.ifPresent(BrAPIUpdate.BrAPIUpdateState::restore); + priorBrAPIObservations.ifPresent(WorkflowUpdate.BrAPIUpdateState::restore); // Undo the prior local transaction return compensatePrior(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index 7b5deb55c..0219f8949 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -18,7 +18,7 @@ public class BrAPIObservationUnitCommit extends ExpUnitMiddleware { private BrAPICreationFactory brAPICreationFactory; private WorkflowCreation brAPIObservationUnitCreation; - private Optional createdBrAPIObservationUnits; + private Optional createdBrAPIObservationUnits; @Inject public BrAPIObservationUnitCommit(BrAPICreationFactory brAPICreationFactory) { @@ -28,7 +28,7 @@ public BrAPIObservationUnitCommit(BrAPICreationFactory brAPICreationFactory) { public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try{ brAPIObservationUnitCreation = brAPICreationFactory.observationUnitWorkflowCreationBean(context); - createdBrAPIObservationUnits = brAPIObservationUnitCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); + createdBrAPIObservationUnits = brAPIObservationUnitCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); } catch (ApiException e) { context.getExpUnitContext().setProcessError(new MiddlewareError(e)); return this.compensate(context); @@ -42,7 +42,7 @@ public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); // Delete any created trials from the BrAPI service - createdBrAPIObservationUnits.ifPresent(BrAPICreation.BrAPICreationState::undo); + createdBrAPIObservationUnits.ifPresent(WorkflowCreation.BrAPICreationState::undo); // Undo the prior local transaction return compensatePrior(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java index b779e23be..ffdf4f180 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -21,7 +21,7 @@ public class BrAPIStudyCommit extends ExpUnitMiddleware { private BrAPICreationFactory brAPICreationFactory; private WorkflowCreation brAPIStudyCreation; - private Optional createdBrAPIStudies; + private Optional createdBrAPIStudies; @Inject public BrAPIStudyCommit(BrAPICreationFactory brAPICreationFactory) { @@ -31,7 +31,7 @@ public BrAPIStudyCommit(BrAPICreationFactory brAPICreationFactory) { public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { brAPIStudyCreation = brAPICreationFactory.studyWorkflowCreationBean(context); - createdBrAPIStudies = brAPIStudyCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); + createdBrAPIStudies = brAPIStudyCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); } catch (ApiException e) { context.getExpUnitContext().setProcessError(new MiddlewareError(e)); return this.compensate(context); @@ -46,7 +46,7 @@ public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); // Delete any created studies from the BrAPI service - createdBrAPIStudies.ifPresent(BrAPICreation.BrAPICreationState::undo); + createdBrAPIStudies.ifPresent(WorkflowCreation.BrAPICreationState::undo); // Undo the prior local transaction return compensatePrior(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index 23547c0c2..1c59dfc12 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -1,17 +1,13 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; import io.micronaut.context.annotation.Prototype; -import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPITrialCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPITrialUpdate; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIUpdate; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIUpdateFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; @@ -22,25 +18,27 @@ @Slf4j @Prototype public class BrAPITrialCommit extends ExpUnitMiddleware { - private BrAPICreationFactory brAPICreationFactory; + private final BrAPICreationFactory brAPICreationFactory; + private final BrAPIUpdateFactory brAPIUpdateFactory; private WorkflowCreation brAPITrialCreation; - private BrAPITrialUpdate brAPITrialUpdate; - private Optional createdBrAPITrials; - private Optional priorBrAPITrials; - private Optional updatedTrials; + private WorkflowUpdate brAPITrialUpdate; + private Optional createdBrAPITrials; + private Optional priorBrAPITrials; + private Optional updatedTrials; @Inject - public BrAPITrialCommit(BrAPICreationFactory brAPICreationFactory) { + public BrAPITrialCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUpdateFactory brAPIUpdateFactory) { this.brAPICreationFactory = brAPICreationFactory; + this.brAPIUpdateFactory = brAPIUpdateFactory; } @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { brAPITrialCreation = brAPICreationFactory.trialWorkflowCreationBean(context); - createdBrAPITrials = brAPITrialCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); - brAPITrialUpdate = new BrAPITrialUpdate(context); - priorBrAPITrials = brAPITrialUpdate.getBrAPIState().map(s -> (BrAPIUpdate.BrAPIUpdateState) s); - updatedTrials = brAPITrialUpdate.execute().map(s -> (BrAPIUpdate.BrAPIUpdateState) s); + createdBrAPITrials = brAPITrialCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); + brAPITrialUpdate = brAPIUpdateFactory.trialWorkflowUpdateBean(context); + priorBrAPITrials = brAPITrialUpdate.getBrAPIState().map(s -> s); + updatedTrials = brAPITrialUpdate.execute().map(s -> (WorkflowUpdate.BrAPIUpdateState) s); } catch (ApiException e) { context.getExpUnitContext().setProcessError(new MiddlewareError(e)); @@ -56,10 +54,10 @@ public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); // Delete any created trials from the BrAPI service - createdBrAPITrials.ifPresent(BrAPICreation.BrAPICreationState::undo); + createdBrAPITrials.ifPresent(WorkflowCreation.BrAPICreationState::undo); // Revert any changes made to trials in the BrAPI service - priorBrAPITrials.ifPresent(BrAPIUpdate.BrAPIUpdateState::restore); + priorBrAPITrials.ifPresent(WorkflowUpdate.BrAPIUpdateState::restore); // Undo the prior local transaction return compensatePrior(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index 48a2a6ed6..122593523 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -17,7 +17,7 @@ public class LocationCommit extends ExpUnitMiddleware { private BrAPICreationFactory brAPICreationFactory; private WorkflowCreation locationCreation; - private Optional createdLocations; + private Optional createdLocations; @Inject public LocationCommit(BrAPICreationFactory brAPICreationFactory) { @@ -27,7 +27,7 @@ public LocationCommit(BrAPICreationFactory brAPICreationFactory) { public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { locationCreation = brAPICreationFactory.locationWorkflowCreationBean(context); - createdLocations = locationCreation.execute().map(s -> (BrAPICreation.BrAPICreationState) s); + createdLocations = locationCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); } catch (ApiException e) { context.getExpUnitContext().setProcessError(new MiddlewareError(e)); return this.compensate(context); @@ -41,7 +41,7 @@ public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); // Delete any created locations - createdLocations.ifPresent(BrAPICreation.BrAPICreationState::undo); + createdLocations.ifPresent(WorkflowCreation.BrAPICreationState::undo); // Undo the prior local transaction return compensatePrior(context); From 0c7659adceb946e38590cbd9e844e824d2fcc381 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 17 Jun 2024 15:32:25 -0400 Subject: [PATCH 133/203] initialize pending study by ou id map --- .../appendoverwrite/entity/PendingStudy.java | 19 +++++++---- .../process/ImportTableProcess.java | 18 ++++++---- .../experiment/service/StudyService.java | 33 +++++++++---------- .../importer/ExperimentFileImportTest.java | 4 +-- 4 files changed, 42 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java index d30825d7a..5421438c4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java @@ -3,6 +3,7 @@ import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -17,10 +18,7 @@ import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; @Prototype @@ -159,7 +157,7 @@ public void initializeWorkflow(List members) { return; } - // Construct the pending studies from the BrAPI trials + // Construct the pending studies from the BrAPI studies List> pendingStudies = members.stream() .map(s->(BrAPIStudy) s) .map(pio -> studyService.constructPIOFromBrapiStudy(pio, importContext.getProgram())).collect(Collectors.toList()); @@ -168,7 +166,16 @@ public void initializeWorkflow(List members) { Map> pendingStudyByNameNoScope = pendingStudies.stream() .collect(Collectors.toMap(pio -> Utilities.removeProgramKeyAndUnknownAdditionalData(pio.getBrAPIObject().getStudyName(), importContext.getProgram().getKey()), pio -> pio)); - // Add the map to the context for use in processing import + // Construct a hashmap to look up the pending study by the observation unit ID of a unit stored in the BrAPI service + Map> pendingStudyByOUId = cache.getPendingObsUnitByOUId().entrySet().stream() + .collect(Collectors.toMap(e -> e.getKey(), + e -> Optional.ofNullable(e.getValue().getBrAPIObject().getStudyName()) + .map(studyNameScoped -> Utilities.removeProgramKeyAndUnknownAdditionalData(studyNameScoped, importContext.getProgram().getKey())) + .map(nameNoScope -> pendingStudyByNameNoScope.get(nameNoScope)) + .orElseThrow(() -> new IllegalStateException("Observation unit missing study name: " + e.getKey())))); + + // Add the maps to the context for use in processing import cache.setStudyByNameNoScope(pendingStudyByNameNoScope); + cache.setPendingStudyByOUId(pendingStudyByOUId); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 379b74df7..1e2dad6a7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -98,7 +98,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Construct validation errors for any timestamp columns that don't have a matching variable column List importRows = context.getImportContext().getImportRows(); - ValidationErrors validationErrors = context.getPendingData().getValidationErrors(); + ValidationErrors validationErrors = context.getExpUnitContext().getValidationErrors(); List tsValErrs = observationVariableService.validateMatchedTimestamps(varNames, timestampCols).orElse(new ArrayList<>()); for (int i = 0; i < importRows.size(); i++) { int rowNum = i; @@ -115,7 +115,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { Map> tsColByPheno = timestampCols.stream().collect(Collectors.toMap(col -> col.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), col -> col)); // Add the map to the context for use in processing import - context.getPendingData().setTimeStampColByPheno(tsColByPheno); + context.getExpUnitContext().setTimeStampColByPheno(tsColByPheno); // Fetch the traits named in the observation variable columns Program program = context.getImportContext().getProgram(); @@ -135,9 +135,9 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { List sortedTraits = experimentUtil.sortByField(List.copyOf(varNames), new ArrayList<>(traits), TraitEntity::getObservationVariableName); // Get the pending observation dataset - PendingImportObject pendingTrial = ExperimentUtilities.getSingleEntryValue(context.getPendingData().getTrialByNameNoScope()).orElseThrow(()->new UnprocessableEntityException(MULTIPLE_EXP_TITLES.getValue())); + PendingImportObject pendingTrial = ExperimentUtilities.getSingleEntryValue(context.getExpUnitContext().getTrialByNameNoScope()).orElseThrow(()->new UnprocessableEntityException(MULTIPLE_EXP_TITLES.getValue())); String datasetName = String.format("Observation Dataset [%s-%s]", program.getKey(), pendingTrial.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString()); - PendingImportObject pendingDataset = context.getPendingData().getObsVarDatasetByName().get(datasetName); + PendingImportObject pendingDataset = context.getExpUnitContext().getObsVarDatasetByName().get(datasetName); // Add new phenotypes to the pending observation dataset list (NOTE: "obsVarName [programKey]" is used instead of obsVarDbId) // TODO: Change to using actual dbIds as per the BrAPI spec, instead of namespaced obsVar names (necessary for Breedbase) @@ -160,7 +160,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Construct helper lookup tables to use for hashing stored observation data Map unitNameByDbId = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, BrAPIObservationUnit::getObservationUnitName)); Map variableNameByDbId = sortedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableDbId, Trait::getObservationVariableName)); - Map studyNameByDbId = context.getPendingData().getStudyByNameNoScope().values().stream() + Map studyNameByDbId = context.getExpUnitContext().getStudyByNameNoScope().values().stream() .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) .map(PendingImportObject::getBrAPIObject) .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); @@ -173,7 +173,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { }, o->o)); // Add the observation data map to the context for use in processing import - context.getPendingData().setExistingObsByObsHash(observationByObsHash); + context.getExpUnitContext().setExistingObsByObsHash(observationByObsHash); // Build new pending observation data for each phenotype Map> pendingObservationByHash = new HashMap<>(); @@ -184,6 +184,10 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); // Construct the pending import for the row + Optional.ofNullable(context.getImportContext().getMappedBrAPIImport()).orElseGet(() -> { + context.getImportContext().setMappedBrAPIImport(new HashMap<>()); + return new HashMap<>(); + }); PendingImport mappedImportRow = context.getImportContext().getMappedBrAPIImport().getOrDefault(rowNum, new PendingImport()); String unitId = row.getObsUnitID(); mappedImportRow.setTrial(context.getExpUnitContext().getPendingTrialByOUId().get(unitId)); @@ -305,7 +309,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { } // Add the pending observation map to the context for use in processing the import - context.getPendingData().setPendingObservationByHash(pendingObservationByHash); + context.getExpUnitContext().setPendingObservationByHash(pendingObservationByHash); return processNext(context); } catch (DoesNotExistException | ApiException | UnprocessableEntityException | ValidatorException e) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index 0a162aba0..e035ecc09 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -222,23 +222,22 @@ public String getStudyDbIdBelongingToPendingUnit(PendingImportObject> mapPendingStudyByOUId( -// String unitId, -// BrAPIObservationUnit unit, -// Map> studyByName, -// Map> studyByOUId, -// Program program -// ) { -// if (unit.getStudyName() != null) { -// String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); -// studyByOUId.put(unitId, studyByName.get(studyName)); -// } else { -// throw new IllegalStateException("Observation unit missing study name: " + unitId); -// } -// -// return studyByOUId; -// } + public Map> mapPendingStudyByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> studyByName, + Map> studyByOUId, + Program program + ) { + if (unit.getStudyName() != null) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); + studyByOUId.put(unitId, studyByName.get(studyName)); + } else { + throw new IllegalStateException("Observation unit missing study name: " + unitId); + } + + return studyByOUId; + } // TODO: used by expunit workflow // private PendingImportObject fetchOrCreateStudyPIO( diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 5f037c4af..43ef78140 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1038,7 +1038,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - String workflowId = "append-dataset"; + String workflowId = "new-experiment"; importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, workflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); @@ -1068,7 +1068,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - + workflowId = "append-dataset"; JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, workflowId); //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); From 5a14c3173056394d0a4f7dde9103e313948fc744 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 17 Jun 2024 17:15:33 -0400 Subject: [PATCH 134/203] initialize pending trial by ou id --- .../appendoverwrite/entity/PendingStudy.java | 2 +- .../appendoverwrite/entity/PendingTrial.java | 28 +++++++++++++++- .../experiment/service/StudyService.java | 32 +++++++++---------- .../experiment/service/TrialService.java | 5 +++ 4 files changed, 49 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java index 5421438c4..2beb33a7b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java @@ -168,7 +168,7 @@ public void initializeWorkflow(List members) { // Construct a hashmap to look up the pending study by the observation unit ID of a unit stored in the BrAPI service Map> pendingStudyByOUId = cache.getPendingObsUnitByOUId().entrySet().stream() - .collect(Collectors.toMap(e -> e.getKey(), + .collect(Collectors.toMap(Map.Entry::getKey, e -> Optional.ofNullable(e.getValue().getBrAPIObject().getStudyName()) .map(studyNameScoped -> Utilities.removeProgramKeyAndUnknownAdditionalData(studyNameScoped, importContext.getProgram().getKey())) .map(nameNoScope -> pendingStudyByNameNoScope.get(nameNoScope)) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java index c76b03d0f..db777460a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java @@ -181,7 +181,33 @@ public void initializeWorkflow(List members) { Map> pendingTrialByNameNoScope = pendingTrials.stream() .collect(Collectors.toMap(pio -> Utilities.removeProgramKey(pio.getBrAPIObject().getTrialName(), importContext.getProgram().getKey()), pio -> pio)); - // Add the map to the context for use in processing import + // Construct a hashmap to look up the pending trial by the observation unit ID of a unit stored in the BrAPI service + Map> pendingTrialByOUId = cache.getPendingObsUnitByOUId().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> { + String trialName = e.getValue().getBrAPIObject().getTrialName(); + String studyName = e.getValue().getBrAPIObject().getStudyName(); + + if (trialName != null) { + String nameNoScope = Utilities.removeProgramKeyAndUnknownAdditionalData(trialName, importContext.getProgram().getKey()); + return Optional.ofNullable(pendingTrialByNameNoScope.get(nameNoScope)) + .orElseThrow(() -> new IllegalStateException("Failed to find pending trial for observation unit" + e.getKey())); + } else if (studyName != null) { + String nameNoScope = Utilities.removeProgramKeyAndUnknownAdditionalData( + cache.getStudyByNameNoScope().get(studyName).getBrAPIObject().getTrialName(), + importContext.getProgram().getKey() + ); + return Optional.ofNullable(pendingTrialByNameNoScope.get(nameNoScope)) + .orElseThrow(() -> new IllegalStateException("Failed to find pending trial for observation unit" + e.getKey())); + } else { + throw new IllegalStateException("Observation Unit missing trial name and study name: " + e.getKey()); + } + } + )); + + // Add the maps to the context for use in processing import cache.setTrialByNameNoScope(pendingTrialByNameNoScope); + cache.setPendingTrialByOUId(pendingTrialByOUId); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index e035ecc09..ee288a404 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -222,22 +222,22 @@ public String getStudyDbIdBelongingToPendingUnit(PendingImportObject> mapPendingStudyByOUId( - String unitId, - BrAPIObservationUnit unit, - Map> studyByName, - Map> studyByOUId, - Program program - ) { - if (unit.getStudyName() != null) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); - studyByOUId.put(unitId, studyByName.get(studyName)); - } else { - throw new IllegalStateException("Observation unit missing study name: " + unitId); - } - - return studyByOUId; - } +// public Map> mapPendingStudyByOUId( +// String unitId, +// BrAPIObservationUnit unit, +// Map> studyByName, +// Map> studyByOUId, +// Program program +// ) { +// if (unit.getStudyName() != null) { +// String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); +// studyByOUId.put(unitId, studyByName.get(studyName)); +// } else { +// throw new IllegalStateException("Observation unit missing study name: " + unitId); +// } +// +// return studyByOUId; +// } // TODO: used by expunit workflow // private PendingImportObject fetchOrCreateStudyPIO( diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index fe68e61c3..20acea259 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -329,6 +329,11 @@ private Map> initializeTrialByNameNoScop private void initializeTrialsForExistingObservationUnits(Program program, Map> observationUnitByNameNoScope, Map> trialByName) { } + public PendingImportObject getPendingTrialByNameNoScope(Map> pendingTrialByNameNoScope, String nameNoScope) { + return Optional.ofNullable(pendingTrialByNameNoScope.get(nameNoScope)) + .orElseThrow(() -> new IllegalStateException("Failed to find pending trial for observation unit")); + } + // TODO: used by expunit workflow public Map> mapPendingTrialByOUId( String unitId, From c5c0a2924ff07ab793911cc9355a3bb5bd1e579d Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 17 Jun 2024 17:49:22 -0400 Subject: [PATCH 135/203] initialize pending location by ou id --- .../entity/PendingLocation.java | 27 +++++++++++++++---- .../experiment/service/TrialService.java | 5 ---- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java index 64c40bef6..e9e12ffc8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java @@ -2,6 +2,7 @@ import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -18,10 +19,7 @@ import org.breedinginsight.services.exceptions.UnprocessableEntityException; import javax.inject.Inject; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; @Prototype @@ -173,7 +171,26 @@ public void initializeWorkflow(List members) { // Construct a hashmap to look up the pending location by location name Map> pendingLocationByName = pendingLocations.stream().collect(Collectors.toMap(pio -> pio.getBrAPIObject().getName(), pio -> pio)); - // Add the map to the context for use in processing import + // Construct a hashmap to look up the pending location by the observation unit ID of a unit stored in the BrAPI service + Map> pendingLocationByOUId = cache.getPendingObsUnitByOUId().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> { + String name = Optional.ofNullable(e.getValue().getBrAPIObject().getLocationName()) + .orElseGet(() -> { + PendingImportObject pendingStudy = cache.getPendingStudyByOUId().get(e.getKey()); + if (pendingStudy == null) { + throw new IllegalStateException("Observation unit missing study: " + e.getKey()); + } + return pendingStudy.getBrAPIObject().getLocationName(); + }); + return Optional.ofNullable(pendingLocationByName.get(name)) + .orElseThrow(() -> new IllegalStateException("Observation unit missing location: " + e.getKey())); + } + )); + + // Add the maps to the context for use in processing import cache.setLocationByName(pendingLocationByName); + cache.setPendingLocationByOUId(pendingLocationByOUId); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index 20acea259..fe68e61c3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -329,11 +329,6 @@ private Map> initializeTrialByNameNoScop private void initializeTrialsForExistingObservationUnits(Program program, Map> observationUnitByNameNoScope, Map> trialByName) { } - public PendingImportObject getPendingTrialByNameNoScope(Map> pendingTrialByNameNoScope, String nameNoScope) { - return Optional.ofNullable(pendingTrialByNameNoScope.get(nameNoScope)) - .orElseThrow(() -> new IllegalStateException("Failed to find pending trial for observation unit")); - } - // TODO: used by expunit workflow public Map> mapPendingTrialByOUId( String unitId, From b69939dcdac78524a25c6204c42f02843bf4daa1 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 17 Jun 2024 18:14:04 -0400 Subject: [PATCH 136/203] initialize pending dataset by ou id --- .../appendoverwrite/entity/PendingDataset.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java index 1a588d186..1390c2d4b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java @@ -187,7 +187,22 @@ public void initializeWorkflow(List members) { Map> pendingDatasetByName = pendingDatasets.stream() .collect(Collectors.toMap(pio -> pio.getBrAPIObject().getListName(),pio -> pio)); - // Add the map to the context for use in processing import + // Construct a hashmap to look up the pending location by the observation unit ID of a unit stored in the BrAPI service + Map> pendingObsDatasetByOUId = cache.getPendingObsUnitByOUId().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> { + if (cache.getPendingTrialByOUId().isEmpty() || + cache.getObsVarDatasetByName().isEmpty() || + !cache.getPendingTrialByOUId().values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + throw new IllegalStateException("There is not an observation data set for this unit: " + e.getKey()); + } + return cache.getObsVarDatasetByName().values().iterator().next(); + } + )); + + // Add the maps to the context for use in processing import cache.setObsVarDatasetByName(pendingDatasetByName); + cache.setPendingObsDatasetByOUId(pendingObsDatasetByOUId); } } From d34009585859b2e9d050e7ea1d03af7ae217969a Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 17 Jun 2024 18:27:49 -0400 Subject: [PATCH 137/203] initialize pending germplasm by ou id --- .../appendoverwrite/entity/PendingDataset.java | 2 +- .../appendoverwrite/entity/PendingGermplasm.java | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java index 1390c2d4b..0d6299552 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java @@ -187,7 +187,7 @@ public void initializeWorkflow(List members) { Map> pendingDatasetByName = pendingDatasets.stream() .collect(Collectors.toMap(pio -> pio.getBrAPIObject().getListName(),pio -> pio)); - // Construct a hashmap to look up the pending location by the observation unit ID of a unit stored in the BrAPI service + // Construct a hashmap to look up the pending dataset by the observation unit ID of a unit stored in the BrAPI service Map> pendingObsDatasetByOUId = cache.getPendingObsUnitByOUId().entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java index 8dc8b0e6f..3b01581bd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java @@ -3,6 +3,7 @@ import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; @@ -136,7 +137,19 @@ public void initializeWorkflow(List members) { // Construct a hashmap to look up the pending germplasm by gid Map> pendingGermplasmByGID = pendingGermplasm.stream().collect(Collectors.toMap(germplasmService::getGIDFromGermplasmPIO, pio -> pio)); - // Add the map to the context for use in processing import + // Construct a hashmap to look up the pending germplasm by the observation unit ID of a unit stored in the BrAPI service + Map> pendingGermplasmByOUId = cache.getPendingObsUnitByOUId().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> { + String gid = Optional.ofNullable(e.getValue().getBrAPIObject().getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.GID).getAsString()) + .orElseThrow(() -> new IllegalStateException("GID not set for unit: " + e.getKey())); + return Optional.ofNullable(pendingGermplasmByGID.get(gid)).orElseThrow(() -> new IllegalStateException("Observation unit missing germplasm: " + e.getKey())); + } + )); + + // Add the maps to the context for use in processing import cache.setExistingGermplasmByGID(pendingGermplasmByGID); + cache.setPendingGermplasmByOUId(pendingGermplasmByOUId); } } From 9b44987dc404013f56601c50b109016651de0372 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 17 Jun 2024 23:46:55 -0400 Subject: [PATCH 138/203] create processed data factory --- .../entity/PendingDataset.java | 4 +- .../process/ImportTableProcess.java | 17 ++- .../middleware/process/InitialData.java | 35 ++--- .../middleware/process/OverwrittenData.java | 9 +- .../process/ProcessedDataFactory.java | 120 ++++++++++++++++++ .../middleware/process/UnchangedData.java | 4 + .../{ => misc/brapi}/FieldValidation.java | 2 +- .../brapi}/ImportPreviewStatistics.java | 5 +- .../brapi/NewPendingBrAPIObjects.java | 2 +- 9 files changed, 170 insertions(+), 28 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/{ => misc/brapi}/FieldValidation.java (94%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/{ => misc/brapi}/ImportPreviewStatistics.java (83%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/{ => misc}/brapi/NewPendingBrAPIObjects.java (94%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java index 0d6299552..7cddb69a8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java @@ -193,11 +193,11 @@ public void initializeWorkflow(List members) { Map.Entry::getKey, e -> { if (cache.getPendingTrialByOUId().isEmpty() || - cache.getObsVarDatasetByName().isEmpty() || + pendingDatasetByName.isEmpty() || !cache.getPendingTrialByOUId().values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { throw new IllegalStateException("There is not an observation data set for this unit: " + e.getKey()); } - return cache.getObsVarDatasetByName().values().iterator().next(); + return pendingDatasetByName.values().iterator().next(); } )); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 1e2dad6a7..587f74cb3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -59,6 +59,7 @@ public class ImportTableProcess extends ExpUnitMiddleware { Gson gson; FieldValidator fieldValidator; AppendStatistic statistic; + ProcessedDataFactory processedDataFactory; @Inject public ImportTableProcess(StudyService studyService, @@ -67,7 +68,8 @@ public ImportTableProcess(StudyService studyService, ObservationService observationService, ExperimentUtilities experimentUtil, FieldValidator fieldValidator, - AppendStatistic statistic) { + AppendStatistic statistic, + ProcessedDataFactory processedDataFactory) { this.studyService = studyService; this.observationVariableService = observationVariableService; this.brAPIObservationDAO = brAPIObservationDAO; @@ -76,6 +78,7 @@ public ImportTableProcess(StudyService studyService, this.gson = new Gson(); this.fieldValidator = fieldValidator; this.statistic = statistic; + this.processedDataFactory = processedDataFactory; } @Override @@ -98,6 +101,10 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Construct validation errors for any timestamp columns that don't have a matching variable column List importRows = context.getImportContext().getImportRows(); + Optional.ofNullable(context.getExpUnitContext().getValidationErrors()).orElseGet(() -> { + context.getExpUnitContext().setValidationErrors(new ValidationErrors()); + return new ValidationErrors(); + }); ValidationErrors validationErrors = context.getExpUnitContext().getValidationErrors(); List tsValErrs = observationVariableService.validateMatchedTimestamps(varNames, timestampCols).orElse(new ArrayList<>()); for (int i = 0; i < importRows.size(); i++) { @@ -240,7 +247,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { Trait changeTrait = gson.fromJson(gson.toJson(traitByPhenoColName.get(phenoColumnName)), Trait.class); // Create new instance of OverwrittenData - processedData = new OverwrittenData(canOverwrite, + processedData = processedDataFactory.overwrittenDataBean(canOverwrite, context.getImportContext().isCommit(), unitId, changeTrait, @@ -255,17 +262,17 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { } else { // create new instance of UnchangedData - processedData = new UnchangedData(observation, program); + processedData = processedDataFactory.unchangedDataBean(observation, program); } } else { // Clone the observation unit and trait - BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getExpUnitContext().getPendingObsUnitByOUId().get(row.getExpUnitId()).getBrAPIObject()), BrAPIObservationUnit.class); + BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getExpUnitContext().getPendingObsUnitByOUId().get(row.getObsUnitID()).getBrAPIObject()), BrAPIObservationUnit.class); Trait initialTrait = gson.fromJson(gson.toJson(traitByPhenoColName.get(phenoColumnName)), Trait.class); // create new instance of InitialData - processedData = new InitialData(context.getImportContext().isCommit(), + processedData = processedDataFactory.initialDataBean(context.getImportContext().isCommit(), cellData, phenoColumnName, initialTrait, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java index 03a139e89..37b214269 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java @@ -1,6 +1,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; import com.google.gson.Gson; +import io.micronaut.context.annotation.Prototype; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.model.v1.response.ValidationError; @@ -22,6 +23,7 @@ import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.BRAPI_REFERENCE_SOURCE; +@Prototype public class InitialData extends VisitedObservationData { boolean isCommit; String cellData; @@ -35,23 +37,24 @@ public class InitialData extends VisitedObservationData { BrAPIObservationUnit observationUnit; User user; Program program; - @Inject - FieldValidator fieldValidator; - @Inject - StudyService studyService; + private final FieldValidator fieldValidator; + private final StudyService studyService; Gson gson; - public InitialData(boolean isCommit, - String cellData, - String phenoColumnName, - Trait trait, - ExperimentObservation row, - UUID trialId, - UUID studyId, - String unitId, - String studyYear, - BrAPIObservationUnit observationUnit, User user, - Program program) { + InitialData(boolean isCommit, + String cellData, + String phenoColumnName, + Trait trait, + ExperimentObservation row, + UUID trialId, + UUID studyId, + String unitId, + String studyYear, + BrAPIObservationUnit observationUnit, + User user, + Program program, + FieldValidator fieldValidator, + StudyService studyService) { this.isCommit = isCommit; this.cellData = cellData; this.phenoColumnName = phenoColumnName; @@ -64,6 +67,8 @@ public InitialData(boolean isCommit, this.observationUnit = observationUnit; this.user = user; this.program = program; + this.fieldValidator = fieldValidator; + this.studyService = studyService; this.gson = new Gson(); } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java index 3473151aa..16706c0d2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java @@ -3,6 +3,7 @@ import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonObject; +import io.micronaut.context.annotation.Prototype; import io.micronaut.http.HttpStatus; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.api.model.v1.response.ValidationError; @@ -24,8 +25,9 @@ import java.util.Optional; import java.util.UUID; +@Prototype public class OverwrittenData extends VisitedObservationData { - @Inject + FieldValidator fieldValidator; Gson gson; boolean canOverwrite; @@ -41,6 +43,7 @@ public class OverwrittenData extends VisitedObservationData { UUID userId; Program program; + @Inject public OverwrittenData(boolean canOverwrite, boolean isCommit, String unitId, @@ -52,7 +55,8 @@ public OverwrittenData(boolean canOverwrite, String reason, BrAPIObservation observation, UUID userId, - Program program) { + Program program, + FieldValidator fieldValidator) { this.canOverwrite = canOverwrite; this.isCommit = isCommit; this.unitId = unitId; @@ -65,6 +69,7 @@ public OverwrittenData(boolean canOverwrite, this.observation = observation; this.userId = userId; this.program = program; + this.fieldValidator = fieldValidator; this.gson = new Gson(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java new file mode 100644 index 000000000..e495b844e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java @@ -0,0 +1,120 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import com.google.gson.Gson; +import io.micronaut.context.annotation.Bean; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.model.User; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.BRAPI_REFERENCE_SOURCE; + +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Prototype; + + +@Factory +public class ProcessedDataFactory { + private final FieldValidator fieldValidator; + private final StudyService studyService; + + @Inject + public ProcessedDataFactory(FieldValidator fieldValidator, + StudyService studyService) { + + this.fieldValidator = fieldValidator; + this.studyService = studyService; + } + + public static InitialData initialData(boolean isCommit, + String cellData, + String phenoColumnName, + Trait trait, + ExperimentObservation row, + UUID trialId, + UUID studyId, + String unitId, + String studyYear, + BrAPIObservationUnit observationUnit, + User user, + Program program, + FieldValidator fieldValidator, + StudyService studyService) { + return new InitialData(isCommit, cellData, phenoColumnName, trait, row, trialId, studyId, unitId, studyYear, observationUnit, user, program, fieldValidator, studyService); + } + + public static OverwrittenData overwrittenData(boolean canOverwrite, + boolean isCommit, + String unitId, + Trait trait, + String phenoColumnName, + String timestampColumnName, + String cellData, + String timestamp, + String reason, + BrAPIObservation observation, + UUID userId, + Program program, + FieldValidator fieldValidator) { + return new OverwrittenData(canOverwrite, isCommit, unitId, trait, phenoColumnName, timestampColumnName, cellData, timestamp, reason, observation, userId, program, fieldValidator); + } + + public static UnchangedData unchangedData(BrAPIObservation observation, Program program) { + return new UnchangedData(observation, program); + } + + @Bean + @Prototype + public InitialData initialDataBean(boolean isCommit, + String cellData, + String phenoColumnName, + Trait trait, + ExperimentObservation row, + UUID trialId, + UUID studyId, + String unitId, + String studyYear, + BrAPIObservationUnit observationUnit, + User user, + Program program) { + return initialData(isCommit, cellData, phenoColumnName, trait, row, trialId, studyId, unitId, studyYear, observationUnit, user, program, fieldValidator, studyService); + } + + @Bean + @Prototype + public OverwrittenData overwrittenDataBean(boolean canOverwrite, + boolean isCommit, + String unitId, + Trait trait, + String phenoColumnName, + String timestampColumnName, + String cellData, + String timestamp, + String reason, + BrAPIObservation observation, + UUID userId, + Program program) { + return overwrittenData(canOverwrite, isCommit, unitId, trait, phenoColumnName, timestampColumnName, cellData, timestamp, reason, observation, userId, program, fieldValidator); + } + + @Bean + @Prototype + public UnchangedData unchangedDataBean(BrAPIObservation observation, Program program) { + return unchangedData(observation, program); + } +} + diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java index 6c3245966..8cabd3045 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; +import io.micronaut.context.annotation.Prototype; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -7,13 +8,16 @@ import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; +import javax.inject.Inject; import java.util.List; import java.util.Optional; +@Prototype public class UnchangedData extends VisitedObservationData { BrAPIObservation observation; Program program; + @Inject public UnchangedData(BrAPIObservation observation, Program program) { this.observation = observation; this.program = program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/FieldValidation.java similarity index 94% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/FieldValidation.java index 300da17be..e02f021ce 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/FieldValidation.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.misc.brapi; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/ImportPreviewStatistics.java similarity index 83% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/ImportPreviewStatistics.java index 8847a72a8..5848db383 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/ImportPreviewStatistics.java @@ -1,8 +1,9 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.misc.brapi; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi.NewPendingBrAPIObjects; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.misc.brapi.FieldValidation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.misc.brapi.NewPendingBrAPIObjects; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/NewPendingBrAPIObjects.java similarity index 94% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/NewPendingBrAPIObjects.java index bf1e181f5..275b74e11 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/NewPendingBrAPIObjects.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.misc.brapi; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; From 5cfa4cf8d3b0e6740936e37ba767147fd12550e3 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 18 Jun 2024 10:42:43 -0400 Subject: [PATCH 139/203] fix numeric trait validator --- .../middleware/validator/NumericalValidator.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/NumericalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/NumericalValidator.java index 97dbd8680..13a95062f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/NumericalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/NumericalValidator.java @@ -53,12 +53,12 @@ public Optional validateField(String fieldName, String value, T Optional number = observationService.validNumericValue(value); Optional validationError = number .flatMap(num -> { - if (!observationService.validNumericRange(num, variable.getScale())) { + if (observationService.validNumericRange(num, variable.getScale())) { + return Optional.empty(); // Return empty Optional if value is in numeric range + } else { return Optional.of(new ValidationError(fieldName, "Value outside of min/max range detected", HttpStatus.UNPROCESSABLE_ENTITY)); } - return Optional.empty(); - }) - .or(() -> Optional.of(new ValidationError(fieldName, "Non-numeric text detected", HttpStatus.UNPROCESSABLE_ENTITY))); + }); return validationError; From 9727a1823ef65bb212162879ffd908a43b517996 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 18 Jun 2024 13:56:29 -0400 Subject: [PATCH 140/203] fix initial observation creation to use existing brapi data --- .../experiment/ExperimentUtilities.java | 33 ++++++++++++ .../process/ImportTableProcess.java | 4 +- .../middleware/process/InitialData.java | 38 ++++++++++--- .../process/ProcessedDataFactory.java | 37 +++++++------ .../service/ObservationService.java | 53 +++++++++++++++++++ .../experiment/service/StudyService.java | 40 ++++++++++++++ .../utilities/cache/RedissonManager.java | 26 --------- .../importer/ExperimentFileImportTest.java | 4 +- 8 files changed, 181 insertions(+), 54 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/utilities/cache/RedissonManager.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index dcee2f9e3..72ac579a1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -7,12 +7,14 @@ import io.micronaut.http.exceptions.HttpStatusException; import io.reactivex.functions.Function; import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIStudy; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.model.Program; @@ -188,4 +190,35 @@ public List sortByField(List sortedFields, List unsortedItems, return unsortedItems; } + + public List getBrAPIExternalReferences( + Program program, String referenceSourceBaseName, UUID trialId, UUID datasetId, UUID studyId, UUID obsUnitId, UUID observationId) { + List refs = new ArrayList<>(); + + addReference(refs, program.getId(), referenceSourceBaseName, ExternalReferenceSource.PROGRAMS); + if (trialId != null) { + addReference(refs, trialId, referenceSourceBaseName, ExternalReferenceSource.TRIALS); + } + if (datasetId != null) { + addReference(refs, datasetId, referenceSourceBaseName, ExternalReferenceSource.DATASET); + } + if (studyId != null) { + addReference(refs, studyId, referenceSourceBaseName, ExternalReferenceSource.STUDIES); + } + if (obsUnitId != null) { + addReference(refs, obsUnitId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATION_UNITS); + } + if (observationId != null) { + addReference(refs, observationId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATIONS); + } + + return refs; + } + + private void addReference(List refs, UUID uuid, String referenceBaseNameSource, ExternalReferenceSource refSourceName) { + BrAPIExternalReference reference = new BrAPIExternalReference(); + reference.setReferenceSource(String.format("%s/%s", referenceBaseNameSource, refSourceName.getName())); + reference.setReferenceID(uuid.toString()); + refs.add(reference); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 587f74cb3..8e3fd8544 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -273,13 +273,15 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // create new instance of InitialData processedData = processedDataFactory.initialDataBean(context.getImportContext().isCommit(), + context.getExpUnitContext().getPendingGermplasmByOUId().get(unitId).getBrAPIObject().getGermplasmName(), + context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject(), cellData, phenoColumnName, initialTrait, row, pendingTrial.getId(), context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getId(), - unitId, + UUID.fromString(unitId), context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getSeasons().get(0), observationUnit, context.getImportContext().getUser(), diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java index 37b214269..cab781342 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java @@ -2,6 +2,8 @@ import com.google.gson.Gson; import io.micronaut.context.annotation.Prototype; +import org.brapi.v2.model.core.BrAPISeason; +import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.model.v1.response.ValidationError; @@ -9,6 +11,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; @@ -26,36 +29,44 @@ @Prototype public class InitialData extends VisitedObservationData { boolean isCommit; + String germplasmName; + BrAPIStudy study; String cellData; String phenoColumnName; Trait trait; ExperimentObservation row; UUID trialId; UUID studyId; - String unitId; + UUID unitId; String studyYear; BrAPIObservationUnit observationUnit; User user; Program program; private final FieldValidator fieldValidator; private final StudyService studyService; + private final ObservationService observationService; Gson gson; InitialData(boolean isCommit, + String germplasmName, + BrAPIStudy study, String cellData, String phenoColumnName, Trait trait, ExperimentObservation row, UUID trialId, UUID studyId, - String unitId, + UUID unitId, String studyYear, BrAPIObservationUnit observationUnit, User user, Program program, FieldValidator fieldValidator, - StudyService studyService) { + StudyService studyService, + ObservationService observationService) { this.isCommit = isCommit; + this.germplasmName = germplasmName; + this.study = study; this.cellData = cellData; this.phenoColumnName = phenoColumnName; this.trait = trait; @@ -69,6 +80,7 @@ public class InitialData extends VisitedObservationData { this.program = program; this.fieldValidator = fieldValidator; this.studyService = studyService; + this.observationService = observationService; this.gson = new Gson(); } @Override @@ -83,13 +95,27 @@ public Optional> getValidationErrors() { @Override public PendingImportObject constructPendingObservation() { - String seasonDbId = studyService.seasonDbIdToYear(studyYear, program.getId()); + String seasonDbId = studyService.yearToSeasonDbIdFromDatabase(studyYear, program.getId()); // Generate a new ID for the observation - UUID observationID = UUID.randomUUID(); + UUID observationId = UUID.randomUUID(); // Construct the new observation - BrAPIObservation newObservation = row.constructBrAPIObservation(cellData, phenoColumnName, seasonDbId, observationUnit, isCommit, program, user, BRAPI_REFERENCE_SOURCE, trialId, studyId, UUID.fromString(unitId), observationID); + BrAPIObservation newObservation = observationService.constructNewBrAPIObservation(isCommit, + germplasmName, + phenoColumnName, + study, + seasonDbId, + observationUnit, + cellData, + trialId, + studyId, + unitId, + observationId, + BRAPI_REFERENCE_SOURCE, + user, + program); + //row.constructBrAPIObservation(cellData, phenoColumnName, seasonDbId, observationUnit, isCommit, program, user, BRAPI_REFERENCE_SOURCE, trialId, studyId, UUID.fromString(unitId), observationID); // Construct a pending observation with a status set to NEW return new PendingImportObject<>(ImportObjectState.NEW, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java index e495b844e..1d0c7ab71 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java @@ -1,60 +1,57 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; -import com.google.gson.Gson; import io.micronaut.context.annotation.Bean; +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Prototype; +import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; import org.breedinginsight.model.User; -import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; import java.util.UUID; -import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.BRAPI_REFERENCE_SOURCE; - -import io.micronaut.context.annotation.Factory; -import io.micronaut.context.annotation.Prototype; - @Factory public class ProcessedDataFactory { private final FieldValidator fieldValidator; private final StudyService studyService; + private final ObservationService observationService; @Inject public ProcessedDataFactory(FieldValidator fieldValidator, - StudyService studyService) { + StudyService studyService, + ObservationService observationService) { this.fieldValidator = fieldValidator; this.studyService = studyService; + this.observationService = observationService; } public static InitialData initialData(boolean isCommit, + String germplasmName, + BrAPIStudy study, String cellData, String phenoColumnName, Trait trait, ExperimentObservation row, UUID trialId, UUID studyId, - String unitId, + UUID unitId, String studyYear, BrAPIObservationUnit observationUnit, User user, Program program, FieldValidator fieldValidator, - StudyService studyService) { - return new InitialData(isCommit, cellData, phenoColumnName, trait, row, trialId, studyId, unitId, studyYear, observationUnit, user, program, fieldValidator, studyService); + StudyService studyService, + ObservationService observationService) { + return new InitialData(isCommit, germplasmName, study, cellData, phenoColumnName, trait, row, trialId, studyId, unitId, studyYear, observationUnit, user, program, fieldValidator, studyService, observationService); } public static OverwrittenData overwrittenData(boolean canOverwrite, @@ -80,18 +77,20 @@ public static UnchangedData unchangedData(BrAPIObservation observation, Program @Bean @Prototype public InitialData initialDataBean(boolean isCommit, + String germplasmName, + BrAPIStudy study, String cellData, String phenoColumnName, Trait trait, ExperimentObservation row, UUID trialId, UUID studyId, - String unitId, + UUID unitId, String studyYear, BrAPIObservationUnit observationUnit, User user, Program program) { - return initialData(isCommit, cellData, phenoColumnName, trait, row, trialId, studyId, unitId, studyYear, observationUnit, user, program, fieldValidator, studyService); + return initialData(isCommit, germplasmName, study, cellData, phenoColumnName, trait, row, trialId, studyId, unitId, studyYear, observationUnit, user, program, fieldValidator, studyService, observationService); } @Bean diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index 35275eb20..20726ffb2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -8,14 +8,17 @@ import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPISeason; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.OverwrittenData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; @@ -23,9 +26,11 @@ import org.breedinginsight.model.Program; import org.breedinginsight.model.Scale; import org.breedinginsight.model.Trait; +import org.breedinginsight.model.User; import org.breedinginsight.utilities.Utilities; import tech.tablesaw.columns.Column; +import javax.inject.Inject; import javax.inject.Singleton; import java.math.BigDecimal; import java.time.OffsetDateTime; @@ -36,6 +41,13 @@ @Singleton public class ObservationService { + private final ExperimentUtilities experimentUtilities; + + @Inject + public ObservationService(ExperimentUtilities experimentUtilities) { + this.experimentUtilities = experimentUtilities; + } + public boolean validCategory(List categories, String value) { Set categoryValues = categories.stream() .map(category -> category.getValue().toLowerCase()) @@ -88,6 +100,47 @@ public String getObservationHash(String observationUnitName, String variableName DigestUtils.sha256Hex(StringUtils.defaultString(studyName)); return DigestUtils.sha256Hex(concat); } + + public BrAPIObservation constructNewBrAPIObservation(boolean commit, + String germplasmName, + String variableName, + BrAPIStudy study, + String seasonDbId, + BrAPIObservationUnit obsUnit, + String value, + UUID trialId, + UUID studyId, + UUID obsUnitId, + UUID observationId, + String referenceSource, + User user, + Program program) { + BrAPIObservation observation = new BrAPIObservation(); + observation.setGermplasmName(germplasmName); + + observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.STUDY_NAME, Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); + + observation.setObservationVariableName(variableName); + observation.setObservationUnitDbId(obsUnit.getObservationUnitDbId()); + observation.setObservationUnitName(obsUnit.getObservationUnitName()); + observation.setValue(value); + + // The BrApi server needs this. Breedbase does not. + BrAPISeason season = new BrAPISeason(); + season.setSeasonDbId(seasonDbId); + observation.setSeason(season); + + if(commit) { + Map createdBy = new HashMap<>(); + createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_ID, user.getId()); + createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_NAME, user.getName()); + observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_BY, createdBy); + observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_DATE, DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(OffsetDateTime.now())); + + observation.setExternalReferences(experimentUtilities.getBrAPIExternalReferences(program, referenceSource, trialId,null, studyId, obsUnitId, observationId)); + } + return observation; + } // public Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { // Set ouDbIds = new HashSet<>(); // Set variableDbIds = new HashSet<>(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index ee288a404..4304cbc74 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -135,6 +135,46 @@ private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { return (yearInt == null) ? "" : yearInt.toString(); } + public String yearToSeasonDbIdFromDatabase(String year, UUID programId) { + BrAPISeason targetSeason = null; + List seasons; + try { + seasons = this.brAPISeasonDAO.getSeasonsByYear(year, programId); + for (BrAPISeason season : seasons) { + if (null == season.getSeasonName() || season.getSeasonName().isBlank() || season.getSeasonName().equals(year)) { + targetSeason = season; + break; + } + } + if (targetSeason == null) { + BrAPISeason newSeason = new BrAPISeason(); + Integer intYear = null; + if( StringUtils.isNotBlank(year) ){ + intYear = Integer.parseInt(year); + } + newSeason.setYear(intYear); + newSeason.setSeasonName(year); + targetSeason = this.brAPISeasonDAO.addOneSeason(newSeason, programId); + } + + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + log.error(e.getResponseBody(), e); + } + + return (targetSeason == null) ? null : targetSeason.getSeasonDbId(); + } + + public List seasonsFromDatabase(String year, UUID programId) { + List seasons = null; + try { + seasons = this.brAPISeasonDAO.getSeasonsByYear(year, programId); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + } + + return seasons; + } /** * Fetches a list of BrAPI studies by their study database IDs for a given program. * diff --git a/src/main/java/org/breedinginsight/utilities/cache/RedissonManager.java b/src/main/java/org/breedinginsight/utilities/cache/RedissonManager.java deleted file mode 100644 index a4dd299a2..000000000 --- a/src/main/java/org/breedinginsight/utilities/cache/RedissonManager.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.breedinginsight.utilities.cache; - -import io.micronaut.context.annotation.Value; -import org.redisson.Redisson; -import org.redisson.api.RedissonClient; -import org.redisson.config.Config; -import io.micronaut.context.event.ApplicationEventListener; -import io.micronaut.runtime.server.event.ServerShutdownEvent; - -import javax.inject.Singleton; -import java.io.IOException; - -@Singleton -public class RedissonManager implements ApplicationEventListener { - private final RedissonClient redissonClient; - - public RedissonManager(@Value("${redisson.single-server-config}") String configFile) throws IOException { - Config config = Config.fromYAML(RedissonManager.class.getResourceAsStream(configFile)); - this.redissonClient = Redisson.create(config); - } - - @Override - public void onApplicationEvent(ServerShutdownEvent event) { - redissonClient.shutdown(); - } -} diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 43ef78140..055f98d46 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1030,7 +1030,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newExp.put(Columns.EXP_TYPE, "Phenotyping"); newExp.put(Columns.ENV, "New Env"); newExp.put(Columns.ENV_LOCATION, "Location A"); - newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.ENV_YEAR, "2025"); newExp.put(Columns.EXP_UNIT_ID, "a-1"); newExp.put(Columns.REP_NUM, "1"); newExp.put(Columns.BLOCK_NUM, "1"); @@ -1058,7 +1058,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(Columns.EXP_TYPE, "Phenotyping"); newObservation.put(Columns.ENV, "New Env"); newObservation.put(Columns.ENV_LOCATION, "Location A"); - newObservation.put(Columns.ENV_YEAR, "2023"); + newObservation.put(Columns.ENV_YEAR, "2025"); newObservation.put(Columns.EXP_UNIT_ID, "a-1"); newObservation.put(Columns.REP_NUM, "1"); newObservation.put(Columns.BLOCK_NUM, "1"); From 83ee522ba7af08a024c7856e142bfa11b1c7020e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 18 Jun 2024 17:20:32 -0400 Subject: [PATCH 141/203] fix how brapi-reference-url is injected --- .../experiment/ExperimentUtilities.java | 2 +- .../middleware/process/ImportTableProcess.java | 6 +++++- .../middleware/process/InitialData.java | 13 ++++++------- .../process/ProcessedDataFactory.java | 10 ++++++---- .../model/ExpImportProcessConstants.java | 9 +++++++-- .../experiment/service/ObservationService.java | 18 +----------------- 6 files changed, 26 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 72ac579a1..f807095d3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -191,7 +191,7 @@ public List sortByField(List sortedFields, List unsortedItems, return unsortedItems; } - public List getBrAPIExternalReferences( + public List constructBrAPIExternalReferences( Program program, String referenceSourceBaseName, UUID trialId, UUID datasetId, UUID studyId, UUID obsUnitId, UUID observationId) { List refs = new ArrayList<>(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 8e3fd8544..5ee040dd1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -1,6 +1,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; import com.google.gson.Gson; +import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.map.CaseInsensitiveMap; @@ -51,6 +52,8 @@ @Slf4j @Prototype public class ImportTableProcess extends ExpUnitMiddleware { + @Property(name = "brapi.server.reference-source") + String brapiReferenceSource; StudyService studyService; ObservationVariableService observationVariableService; ObservationService observationService; @@ -272,7 +275,8 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { Trait initialTrait = gson.fromJson(gson.toJson(traitByPhenoColName.get(phenoColumnName)), Trait.class); // create new instance of InitialData - processedData = processedDataFactory.initialDataBean(context.getImportContext().isCommit(), + processedData = processedDataFactory.initialDataBean(brapiReferenceSource, + context.getImportContext().isCommit(), context.getExpUnitContext().getPendingGermplasmByOUId().get(unitId).getBrAPIObject().getGermplasmName(), context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject(), cellData, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java index cab781342..83e48dfdb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java @@ -1,8 +1,8 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; import com.google.gson.Gson; +import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; -import org.brapi.v2.model.core.BrAPISeason; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; @@ -18,16 +18,14 @@ import org.breedinginsight.model.User; import org.breedinginsight.utilities.Utilities; -import javax.inject.Inject; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.UUID; -import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.BRAPI_REFERENCE_SOURCE; - @Prototype public class InitialData extends VisitedObservationData { + String brapiReferenceSource; boolean isCommit; String germplasmName; BrAPIStudy study; @@ -47,7 +45,8 @@ public class InitialData extends VisitedObservationData { private final ObservationService observationService; Gson gson; - InitialData(boolean isCommit, + InitialData(String brapiReferenceSource, + boolean isCommit, String germplasmName, BrAPIStudy study, String cellData, @@ -64,6 +63,7 @@ public class InitialData extends VisitedObservationData { FieldValidator fieldValidator, StudyService studyService, ObservationService observationService) { + this.brapiReferenceSource = brapiReferenceSource; this.isCommit = isCommit; this.germplasmName = germplasmName; this.study = study; @@ -112,10 +112,9 @@ public PendingImportObject constructPendingObservation() { studyId, unitId, observationId, - BRAPI_REFERENCE_SOURCE, + brapiReferenceSource, user, program); - //row.constructBrAPIObservation(cellData, phenoColumnName, seasonDbId, observationUnit, isCommit, program, user, BRAPI_REFERENCE_SOURCE, trialId, studyId, UUID.fromString(unitId), observationID); // Construct a pending observation with a status set to NEW return new PendingImportObject<>(ImportObjectState.NEW, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java index 1d0c7ab71..ff6c17a14 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java @@ -34,7 +34,8 @@ public ProcessedDataFactory(FieldValidator fieldValidator, this.observationService = observationService; } - public static InitialData initialData(boolean isCommit, + public static InitialData initialData(String brapiReferenceSource, + boolean isCommit, String germplasmName, BrAPIStudy study, String cellData, @@ -51,7 +52,7 @@ public static InitialData initialData(boolean isCommit, FieldValidator fieldValidator, StudyService studyService, ObservationService observationService) { - return new InitialData(isCommit, germplasmName, study, cellData, phenoColumnName, trait, row, trialId, studyId, unitId, studyYear, observationUnit, user, program, fieldValidator, studyService, observationService); + return new InitialData(brapiReferenceSource, isCommit, germplasmName, study, cellData, phenoColumnName, trait, row, trialId, studyId, unitId, studyYear, observationUnit, user, program, fieldValidator, studyService, observationService); } public static OverwrittenData overwrittenData(boolean canOverwrite, @@ -76,7 +77,8 @@ public static UnchangedData unchangedData(BrAPIObservation observation, Program @Bean @Prototype - public InitialData initialDataBean(boolean isCommit, + public InitialData initialDataBean(String brapiReferenceSource, + boolean isCommit, String germplasmName, BrAPIStudy study, String cellData, @@ -90,7 +92,7 @@ public InitialData initialDataBean(boolean isCommit, BrAPIObservationUnit observationUnit, User user, Program program) { - return initialData(isCommit, germplasmName, study, cellData, phenoColumnName, trait, row, trialId, studyId, unitId, studyYear, observationUnit, user, program, fieldValidator, studyService, observationService); + return initialData(brapiReferenceSource, isCommit, germplasmName, study, cellData, phenoColumnName, trait, row, trialId, studyId, unitId, studyYear, observationUnit, user, program, fieldValidator, studyService, observationService); } @Bean diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java index 0fc8b807c..72c0545b6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java @@ -2,15 +2,20 @@ import com.fasterxml.jackson.annotation.JsonValue; import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Value; +import lombok.extern.slf4j.Slf4j; +import javax.annotation.PostConstruct; + +@Slf4j public class ExpImportProcessConstants { public static final CharSequence COMMA_DELIMITER = ","; public static final String TIMESTAMP_PREFIX = "TS:"; public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; - public static final String MIDNIGHT = "T00:00:00-00:00"; - @Property(name = "brapi.server.reference-source") public static String BRAPI_REFERENCE_SOURCE; + public static final String MIDNIGHT = "T00:00:00-00:00"; + public enum ErrMessage { MULTIPLE_EXP_TITLES("File contains more than one Experiment Title"), MISSING_OBS_UNIT_ID_ERROR("Experimental entities are missing ObsUnitIDs"), diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index 20726ffb2..5bd097b8a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -1,34 +1,18 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.service; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import io.micronaut.http.server.exceptions.InternalServerException; -import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; -import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPISeason; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.OverwrittenData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.Scale; -import org.breedinginsight.model.Trait; import org.breedinginsight.model.User; import org.breedinginsight.utilities.Utilities; -import tech.tablesaw.columns.Column; import javax.inject.Inject; import javax.inject.Singleton; @@ -137,7 +121,7 @@ public BrAPIObservation constructNewBrAPIObservation(boolean commit, observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_BY, createdBy); observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_DATE, DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(OffsetDateTime.now())); - observation.setExternalReferences(experimentUtilities.getBrAPIExternalReferences(program, referenceSource, trialId,null, studyId, obsUnitId, observationId)); + observation.setExternalReferences(experimentUtilities.constructBrAPIExternalReferences(program, referenceSource, trialId,null, studyId, obsUnitId, observationId)); } return observation; } From 7288435ce13798cc44c564aa6cb4ca9fabc63f4e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 18 Jun 2024 20:25:59 -0400 Subject: [PATCH 142/203] add log info statements for commit brapi data --- .../appendoverwrite/middleware/commit/BrAPIDatasetCommit.java | 3 +++ .../middleware/commit/BrAPIObservationCommit.java | 2 ++ .../middleware/commit/BrAPIObservationUnitCommit.java | 3 +++ .../appendoverwrite/middleware/commit/BrAPIStudyCommit.java | 1 + .../appendoverwrite/middleware/commit/BrAPITrialCommit.java | 2 ++ .../appendoverwrite/middleware/commit/LocationCommit.java | 3 +++ 6 files changed, 14 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index 8f6eb46f8..cd13fc498 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -1,6 +1,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; @@ -14,6 +15,7 @@ import javax.inject.Inject; import java.util.Optional; +@Slf4j @Prototype public class BrAPIDatasetCommit extends ExpUnitMiddleware { private BrAPICreationFactory brAPICreationFactory; @@ -29,6 +31,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { datasetCreation = brAPICreationFactory.datasetWorkflowCreationBean(context); + log.info("creating new datasets in the BrAPI service"); createdDatasets = datasetCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); } catch (ApiException e) { context.getExpUnitContext().setProcessError(new MiddlewareError(e)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java index c4d33a79f..2220f5a6c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -35,9 +35,11 @@ public BrAPIObservationCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUp public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { brAPIObservationCreation = brAPICreationFactory.observationWorkflowCreationBean(context); + log.info("creating new observations in the BrAPI service"); createdBrAPIObservations = brAPIObservationCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); brAPIObservationUpdate = brAPIUpdateFactory.observationWorkflowUpdateBean(context); priorBrAPIObservations = brAPIObservationUpdate.getBrAPIState().map(s -> s); + log.info("updating existing observations in the BrAPI service"); updatedObservations = brAPIObservationUpdate.execute().map(s -> (WorkflowUpdate.BrAPIUpdateState) s); } catch (ApiException e) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index 0219f8949..48493dbbe 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -1,6 +1,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; @@ -14,6 +15,7 @@ import javax.inject.Inject; import java.util.Optional; +@Slf4j @Prototype public class BrAPIObservationUnitCommit extends ExpUnitMiddleware { private BrAPICreationFactory brAPICreationFactory; @@ -28,6 +30,7 @@ public BrAPIObservationUnitCommit(BrAPICreationFactory brAPICreationFactory) { public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try{ brAPIObservationUnitCreation = brAPICreationFactory.observationUnitWorkflowCreationBean(context); + log.info("creating new observation units in the BrAPI service"); createdBrAPIObservationUnits = brAPIObservationUnitCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); } catch (ApiException e) { context.getExpUnitContext().setProcessError(new MiddlewareError(e)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java index ffdf4f180..ad2819877 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -31,6 +31,7 @@ public BrAPIStudyCommit(BrAPICreationFactory brAPICreationFactory) { public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { brAPIStudyCreation = brAPICreationFactory.studyWorkflowCreationBean(context); + log.info("creating new studies in the BrAPI service"); createdBrAPIStudies = brAPIStudyCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); } catch (ApiException e) { context.getExpUnitContext().setProcessError(new MiddlewareError(e)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index 1c59dfc12..40fbb7a47 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -35,9 +35,11 @@ public BrAPITrialCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUpdateFa public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { brAPITrialCreation = brAPICreationFactory.trialWorkflowCreationBean(context); + log.info("creating new trials in the BrAPI service"); createdBrAPITrials = brAPITrialCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); brAPITrialUpdate = brAPIUpdateFactory.trialWorkflowUpdateBean(context); priorBrAPITrials = brAPITrialUpdate.getBrAPIState().map(s -> s); + log.info("updating existing trials in the BrAPI service"); updatedTrials = brAPITrialUpdate.execute().map(s -> (WorkflowUpdate.BrAPIUpdateState) s); } catch (ApiException e) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index 122593523..3c46fafb7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -1,6 +1,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; @@ -13,6 +14,7 @@ import javax.inject.Inject; import java.util.Optional; +@Slf4j @Prototype public class LocationCommit extends ExpUnitMiddleware { private BrAPICreationFactory brAPICreationFactory; @@ -27,6 +29,7 @@ public LocationCommit(BrAPICreationFactory brAPICreationFactory) { public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { try { locationCreation = brAPICreationFactory.locationWorkflowCreationBean(context); + log.info("creating new locationss in the Deltabreed database"); createdLocations = locationCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); } catch (ApiException e) { context.getExpUnitContext().setProcessError(new MiddlewareError(e)); From ba6285c4449a9945a6290a65335496a53a788e7b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 18 Jun 2024 23:28:02 -0400 Subject: [PATCH 143/203] inject status service into append workflow --- .../AppendOverwritePhenotypesWorkflow.java | 56 ++++++++++++++----- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 753144f9e..5c3f47404 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -1,11 +1,14 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.ImportStatusService; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.Transaction; @@ -19,29 +22,32 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.lang.reflect.Constructor; import java.util.ArrayList; import java.util.Optional; - +@Slf4j @Getter @Singleton public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow { private final ExperimentWorkflowNavigator.Workflow workflow; - private final ExpUnitMiddleware middleware; + private final ExpUnitMiddleware importPreviewMiddleware; + private final ExpUnitMiddleware brapiCommitMiddleware; + private final ImportStatusService statusService; @Inject public AppendOverwritePhenotypesWorkflow(Transaction transaction, ExpUnitIDValidation expUnitIDValidation, WorkflowInitialization workflowInitialization, ImportTableProcess importTableProcess, - BrAPICommit brAPICommit){ + BrAPICommit brAPICommit, + ImportStatusService statusService){ + this.statusService = statusService; this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE; - this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( + this.importPreviewMiddleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( transaction, expUnitIDValidation, workflowInitialization, - importTableProcess, - brAPICommit); + importTableProcess); + this.brapiCommitMiddleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(brAPICommit); } @Override @@ -82,8 +88,8 @@ public Optional process(ImportServiceContext context) { .expUnitContext(new ExpUnitContext()) .build(); - // Process the workflow - ExpUnitMiddlewareContext processedContext = this.middleware.process(workflowContext); + // Process the import preview + ExpUnitMiddlewareContext processedPreviewContext = this.importPreviewMiddleware.process(workflowContext); // TODO: Rethrow any exceptions caught during processing the context // Optional.ofNullable(processedContext.getExpUnitContext().getMiddlewareError()).ifPresent(e -> { @@ -92,14 +98,38 @@ public Optional process(ImportServiceContext context) { // throw newException; // }); - // Shape and return the workflow response + // BUild and return the preview response ImportPreviewResponse response = new ImportPreviewResponse(); - response.setStatistics(processedContext.getExpUnitContext().getStatistic().constructPreviewMap()); - response.setRows(new ArrayList<>(processedContext.getImportContext().getMappedBrAPIImport().values())); - response.setDynamicColumnNames(processedContext.getImportContext().getUpload().getDynamicColumnNamesList()); + response.setStatistics(processedPreviewContext.getExpUnitContext().getStatistic().constructPreviewMap()); + response.setRows(new ArrayList<>(processedPreviewContext.getImportContext().getMappedBrAPIImport().values())); + response.setDynamicColumnNames(processedPreviewContext.getImportContext().getUpload().getDynamicColumnNamesList()); result.ifPresent(importWorkflowResult -> importWorkflowResult.setImportPreviewResponse(Optional.of(response))); + log.debug("Finished mapping data to brapi objects"); + statusService.updateMappedData(context.getUpload(), response, "Finished mapping data to brapi objects"); + + if (!context.isCommit()) { + statusService.updateOk(context.getUpload()); + return result; + } else { + + // get total number of new brapi objects to create + long totalObjects = response.getStatistics().values().stream() + .mapToLong(ImportPreviewStatistics::getNewObjectCount) // Extract newObjectCount from each ImportStatistics entry + .sum(); // Sum the newObjectCount values + log.debug("Starting upload to brapi service"); + statusService.startUpload(context.getUpload(), totalObjects, "Starting upload to brapi service"); + log.debug("Creating new objects in brapi service"); + statusService.updateMessage(context.getUpload(), "Creating new objects in brapi service"); + + // Commit the changes from the processed import preview to the BrAPI service + ExpUnitMiddlewareContext brapiCommittedContext = this.brapiCommitMiddleware.process(processedPreviewContext); + + log.debug("Completed upload to brapi service"); + statusService.finishUpload(context.getUpload(), totalObjects, "Completed upload to brapi service"); + } + return result; } From 6e78eac13d228357552e3f9dab52ecfbb994947f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 20 Jun 2024 12:08:57 -0400 Subject: [PATCH 144/203] update append-dataset integration tests to call workflow --- .../brapi/v2/services/BrAPITrialService.java | 4 +-- .../AppendOverwritePhenotypesWorkflow.java | 6 ++-- .../importer/ExperimentFileImportTest.java | 29 +++++++++---------- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index caab88879..6b6fcbcae 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -390,8 +390,8 @@ private void addObsVarDataToRow( Trait var, Program program) { String varName = Utilities.removeProgramKey(obs.getObservationVariableName(), program.getKey()); - if (!(obs.getValue().equalsIgnoreCase("NA")) && (var.getScale().getDataType().equals(DataType.NUMERICAL) || - var.getScale().getDataType().equals(DataType.DURATION))) { + if (!("NA".equalsIgnoreCase(obs.getValue())) && (DataType.NUMERICAL.equals(var.getScale().getDataType()) || + DataType.DURATION.equals(var.getScale().getDataType()))) { row.put(varName, Double.parseDouble(obs.getValue())); } else { row.put(varName, obs.getValue()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 5c3f47404..ece1e8672 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -52,7 +52,7 @@ public AppendOverwritePhenotypesWorkflow(Transaction transaction, @Override public Optional process(ImportServiceContext context) { - // Workflow processing the context + // Metadata about this workflow processing the context ImportWorkflow workflow = ImportWorkflow.builder() .id(getWorkflow().getId()) .name(getWorkflow().getName()) @@ -60,7 +60,7 @@ public Optional process(ImportServiceContext context) { // No-preview result Optional result = Optional.of(ImportWorkflowResult.builder() - .workflow(workflow) + .workflow(workflow) // attach metadata of this workflow to response .importPreviewResponse(Optional.empty()) .build()); @@ -69,7 +69,7 @@ public Optional process(ImportServiceContext context) { return Optional.empty(); } - // Skip processing if no context, but return no-preview result for this workflow + // Skip processing if no context, but return no-preview result with metadata for this workflow if (context == null) { return result; } diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 055f98d46..13e94dee9 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -86,6 +86,8 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class ExperimentFileImportTest extends BrAPITest { private static final String OVERWRITE = "overwrite"; + private static final String APPEND_OVERWRITE_WORKFLOW = "append-dataset"; + private static final String NEW_EXPERIMENT_WORKFLOW = "new-experiment"; private FannyPack securityFp; private String mappingId; @@ -659,7 +661,7 @@ public void importNewObsVarExistingOu() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, NEW_EXPERIMENT_WORKFLOW); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -687,7 +689,7 @@ public void importNewObsVarExistingOu() { newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId, APPEND_OVERWRITE_WORKFLOW); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -723,7 +725,7 @@ public void importNewObsVarByObsUnitId() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, NEW_EXPERIMENT_WORKFLOW); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -738,7 +740,7 @@ public void importNewObsVarByObsUnitId() { newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId, APPEND_OVERWRITE_WORKFLOW); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -775,7 +777,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); // empty dataset - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, NEW_EXPERIMENT_WORKFLOW); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -803,7 +805,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, commit, client, program, mappingId, APPEND_OVERWRITE_WORKFLOW); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -844,7 +846,7 @@ public void importNewObsExistingOu(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, NEW_EXPERIMENT_WORKFLOW); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -872,7 +874,7 @@ public void importNewObsExistingOu(boolean commit) { newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, APPEND_OVERWRITE_WORKFLOW); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -1038,8 +1040,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - String workflowId = "new-experiment"; - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, workflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, NEW_EXPERIMENT_WORKFLOW); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1068,9 +1069,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - workflowId = "append-dataset"; - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, workflowId); - //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, APPEND_OVERWRITE_WORKFLOW); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -1115,7 +1114,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, NEW_EXPERIMENT_WORKFLOW); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1146,7 +1145,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newObservation.put(traits.get(0).getObservationVariableName(), ""); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, APPEND_OVERWRITE_WORKFLOW); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); From 7246ab68b6e8924cfec55fb514c3d1be764552c5 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 20 Jun 2024 15:23:04 -0400 Subject: [PATCH 145/203] add dataset update to commit middleware --- .../action/update/BrAPIUpdateFactory.java | 12 +++++++++ .../action/update/WorkflowUpdate.java | 3 +-- .../entity/PendingDataset.java | 27 ++++++++++++++++++- .../middleware/commit/BrAPIDatasetCommit.java | 18 +++++++++++-- 4 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdateFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdateFactory.java index af3291620..53baa720c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdateFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdateFactory.java @@ -4,6 +4,7 @@ import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Prototype; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; @@ -29,6 +30,11 @@ public static WorkflowUpdate observationWorkflowUpdate(ExpUnit return new WorkflowUpdate(pendingEntityFactory.pendingObservationBean(context)); } + public static WorkflowUpdate datasetWorkflowUpdate(ExpUnitMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowUpdate(pendingEntityFactory.pendingDatasetBean(context)); + } + @Bean @Prototype public WorkflowUpdate trialWorkflowUpdateBean(ExpUnitMiddlewareContext context) { @@ -40,4 +46,10 @@ public WorkflowUpdate trialWorkflowUpdateBean(ExpUnitMiddlewareConte public WorkflowUpdate observationWorkflowUpdateBean(ExpUnitMiddlewareContext context) { return observationWorkflowUpdate(context, pendingEntityFactory); } + + @Bean + @Prototype + public WorkflowUpdate datasetWorkflowUpdateBean(ExpUnitMiddlewareContext context) { + return datasetWorkflowUpdate(context, pendingEntityFactory); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/WorkflowUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/WorkflowUpdate.java index e57a45ddb..b15e2976d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/WorkflowUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/WorkflowUpdate.java @@ -53,8 +53,7 @@ protected Optional saveAndUpdateCache(List members) { return Optional.ofNullable(members).map(changes -> { try { List savedMembers = entity.brapiPut(changes); - // TODO: set updated fields of workflow brapi object - // entity.updateWorkflow(savedMembers); + entity.updateWorkflow(savedMembers); return new BrAPIUpdateState(savedMembers); } catch (ApiException e) { // TODO: add specific error messages to entity service diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java index 7cddb69a8..83f95df92 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java @@ -105,7 +105,32 @@ public List brapiRead() throws ApiException { */ @Override public List brapiPut(List members) throws ApiException, IllegalArgumentException { - return new ArrayList<>(); + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIListDetails.class)) { + return new ArrayList(); + } + + List updatedDatasets = new ArrayList<>(); + for (U member : members) { + BrAPIListDetails obsVarList = (BrAPIListDetails) member; + String obsVarListDbId = obsVarList.getListDbId(); + + // Get the current observation variables for the dataset from the BrAPI service + List existingObsVarIds = brAPIListDAO.getListById(obsVarListDbId, importContext.getProgram().getId()).getResult().getData(); + + // Find any observation variables that need to be added to the list in the BrAPI service + List newObsVarIds = obsVarList + .getData() + .stream() + .filter(obsVarId -> !existingObsVarIds.contains(obsVarId)).collect(Collectors.toList()); + + // Save the additions to the list in the BrAPI service + List obsVarIds = new ArrayList<>(existingObsVarIds); + obsVarIds.addAll(newObsVarIds); + obsVarList.setData(obsVarIds); + brAPIListDAO.updateBrAPIList(obsVarListDbId, obsVarList, importContext.getProgram().getId()); + } + + return updatedDatasets; } /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index cd13fc498..36f87f4a5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -8,6 +8,8 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPIDatasetCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIUpdateFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; @@ -18,13 +20,18 @@ @Slf4j @Prototype public class BrAPIDatasetCommit extends ExpUnitMiddleware { - private BrAPICreationFactory brAPICreationFactory; + private final BrAPICreationFactory brAPICreationFactory; + private final BrAPIUpdateFactory brAPIUpdateFactory; private WorkflowCreation datasetCreation; + private WorkflowUpdate datasetUpdate; private Optional createdDatasets; + private Optional priorDatasets; + private Optional updatedDatasets; @Inject - public BrAPIDatasetCommit(BrAPICreationFactory brAPICreationFactory) { + public BrAPIDatasetCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUpdateFactory brAPIUpdateFactory) { this.brAPICreationFactory = brAPICreationFactory; + this.brAPIUpdateFactory = brAPIUpdateFactory; } @Override public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { @@ -33,6 +40,10 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { datasetCreation = brAPICreationFactory.datasetWorkflowCreationBean(context); log.info("creating new datasets in the BrAPI service"); createdDatasets = datasetCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); + datasetUpdate = brAPIUpdateFactory.datasetWorkflowUpdateBean(context); + priorDatasets = datasetUpdate.getBrAPIState().map(d -> d); + log.info("adding new observation variables to datasets"); + updatedDatasets = datasetUpdate.execute().map(d -> (WorkflowUpdate.BrAPIUpdateState) d); } catch (ApiException e) { context.getExpUnitContext().setProcessError(new MiddlewareError(e)); return this.compensate(context); @@ -48,6 +59,9 @@ public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { // Delete any created datasets createdDatasets.ifPresent(WorkflowCreation.BrAPICreationState::undo); + // Revert any changes made to datasets in the BrAPI service + priorDatasets.ifPresent(WorkflowUpdate.BrAPIUpdateState::restore); + // Undo the prior local transaction return compensatePrior(context); } From 1e63696f578142caceec56ee8dafb43b06592d11 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 20 Jun 2024 16:52:43 -0400 Subject: [PATCH 146/203] Updated create only tests to use new create workflow --- .../importer/ExperimentFileImportTest.java | 185 +++++++++++++----- .../brapps/importer/ImportTestUtils.java | 69 +++++++ 2 files changed, 203 insertions(+), 51 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 6bc203d64..0a5b4c1e9 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -24,6 +24,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.client.RxHttpClient; import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.netty.cookies.NettyCookie; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; import lombok.SneakyThrows; @@ -77,6 +78,7 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import static io.micronaut.http.HttpRequest.GET; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.*; @@ -147,6 +149,8 @@ public class ExperimentFileImportTest extends BrAPITest { .registerTypeAdapter(BrAPIPagination.class, new PaginationTypeAdapter()) .create(); + private String newExperimentWorkflowId; + @BeforeAll public void setup() { importTestUtils = new ImportTestUtils(); @@ -154,7 +158,27 @@ public void setup() { mappingId = (String) setupObjects.get("mappingId"); testUser = (BiUserEntity) setupObjects.get("testUser"); securityFp = (FannyPack) setupObjects.get("securityFp"); + newExperimentWorkflowId = getNewExperimentWorkflowId(); + } + + /** + * TODO: assumes new workflow is first in list, doesn't look at position property, would be more robust to + * look at that instead of assuming order + * @return + */ + public String getNewExperimentWorkflowId() { + // GET /import/mappings{?importName} + Flowable> call = client.exchange( + GET("/import/mappings/"+mappingId+"/workflows").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + + return JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); } /* @@ -194,16 +218,20 @@ public void importNewExpNewLocNoObsSuccess() { validRow.put(Columns.COLUMN, "1"); validRow.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String workflowId = "new-experiment"; + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId, newExperimentWorkflowId); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + // TODO: remove this + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); @@ -252,16 +280,18 @@ public void importNewExpMultiNewEnvSuccess() { secondEnv.put(Columns.COLUMN, "1"); secondEnv.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(2, previewRows.size()); JsonObject firstRow = previewRows.get(0).getAsJsonObject(); @@ -299,7 +329,8 @@ public void importExistingExpAndEnvErrorMessage() { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + JsonObject expResult = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + //JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); Map dupExp = new HashMap<>(); dupExp.put(Columns.GERMPLASM_GID, "1"); @@ -316,16 +347,17 @@ public void importExistingExpAndEnvErrorMessage() { dupExp.put(Columns.ROW, "1"); dupExp.put(Columns.COLUMN, "1"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, false, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + expResult = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, false, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, expResult.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + expResult); + assertTrue(expResult.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); } @Test @@ -348,9 +380,10 @@ public void importNewEnvNoObsSuccess() { newEnv.put(Columns.ROW, "1"); newEnv.put(Columns.COLUMN, "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); @@ -382,43 +415,53 @@ public void verifyMissingDataThrowsError(boolean commit) { Map noGID = new HashMap<>(base); noGID.remove(Columns.GERMPLASM_GID); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit, newExperimentWorkflowId); Map noExpTitle = new HashMap<>(base); noExpTitle.remove(Columns.EXP_TITLE); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit, newExperimentWorkflowId); Map noExpUnit = new HashMap<>(base); noExpUnit.remove(Columns.EXP_UNIT); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit, newExperimentWorkflowId); Map noExpType = new HashMap<>(base); noExpType.remove(Columns.EXP_TYPE); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit, newExperimentWorkflowId); Map noEnv = new HashMap<>(base); noEnv.remove(Columns.ENV); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit, newExperimentWorkflowId); Map noEnvLoc = new HashMap<>(base); noEnvLoc.remove(Columns.ENV_LOCATION); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); Map noExpUnitId = new HashMap<>(base); noExpUnitId.remove(Columns.EXP_UNIT_ID); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit, newExperimentWorkflowId); Map noExpRep = new HashMap<>(base); noExpRep.remove(Columns.REP_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit, newExperimentWorkflowId); Map noExpBlock = new HashMap<>(base); noExpBlock.remove(Columns.BLOCK_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit, newExperimentWorkflowId); Map noEnvYear = new HashMap<>(base); noEnvYear.remove(Columns.ENV_YEAR); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); } @Test @@ -442,7 +485,8 @@ public void importNewExpWithObsVar() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -492,7 +536,9 @@ public void verifyDiffYearSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); + } @ParameterizedTest @@ -530,7 +576,8 @@ public void verifyDiffLocSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); } @ParameterizedTest @@ -555,7 +602,8 @@ public void importNewExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -595,7 +643,9 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "Red"); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); + } @ParameterizedTest @@ -618,21 +668,24 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + //importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); Map newOU = new HashMap<>(newExp); newOU.put(Columns.EXP_UNIT_ID, "a-2"); newOU.put(Columns.ROW, "1"); newOU.put(Columns.COLUMN, "2"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, true, client, program, mappingId, newExperimentWorkflowId); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); @@ -948,6 +1001,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { - a new experiment is created after the first experiment - verify the second experiment gets created successfully */ + //TODO: this one @Test @SneakyThrows public void importSecondExpAfterFirstExpWithObs() { @@ -969,7 +1023,8 @@ public void importSecondExpAfterFirstExpWithObs() { newExpA.put(Columns.COLUMN, "1"); newExpA.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + //JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + JsonObject resultA = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsA = resultA.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsA.size()); @@ -997,7 +1052,8 @@ public void importSecondExpAfterFirstExpWithObs() { newExpB.put(Columns.COLUMN, "1"); newExpB.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + //JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + JsonObject resultB = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsB = resultB.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsB.size()); @@ -1525,6 +1581,33 @@ private JsonObject uploadAndVerifyFailure(Program program, File file, String exp JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); + assertEquals(1, rowErrors.size()); + JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); + assertEquals(1, fieldErrors.size()); + JsonObject error = fieldErrors.get(0).getAsJsonObject(); + assertEquals(expectedColumnError, error.get("field").getAsString()); + assertEquals(422, error.get("httpStatusCode").getAsInt()); + + return result; + } + + private JsonObject uploadAndVerifyWorkflowFailure(Program program, File file, String expectedColumnError, boolean commit, String workflowId) throws InterruptedException, IOException { + + //Flowable> call = importTestUtils.uploadDataFile(file, null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(file, null, true, client, program, mappingId, newExperimentWorkflowId); + //JsonObject result = JsonParser.parseString(upload).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); assertEquals(1, rowErrors.size()); JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index 12b79ac15..84da9b536 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -97,6 +97,38 @@ public Flowable> uploadDataFile(File file, Map> uploadWorkflowDataFile(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) { + + MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); + + // Upload file + String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", program.getId(), mappingId); + Flowable> call = client.exchange( + POST(uploadUrl, requestBody) + .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + String importId = result.get("importId").getAsString(); + + // Process data + String url = String.format("/programs/%s/import/mappings/%s/workflows/%s/data/%s/%s", program.getId(), mappingId, workflowId, importId, commit ? "commit" : "preview"); + Flowable> processCall = client.exchange( + PUT(url, userData) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + return processCall; + + } + public HttpResponse getUploadedFile(String importId, RxHttpClient client, Program program, String mappingId) throws InterruptedException { Flowable> call = client.exchange( GET(String.format("/programs/%s/import/mappings/%s/data/%s?mapping=true", program.getId(), mappingId, importId)) @@ -170,6 +202,43 @@ public JsonObject uploadAndFetch(File file, Map userData, Boolea return result; } + public JsonObject uploadAndFetchWorkflow(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { + Flowable> call = uploadWorkflowDataFile(file, userData, commit, client, program, mappingId, workflowId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + return result; + } + + public JsonObject uploadAndFetchWorkflowNoStatusCheck(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { + Flowable> call = uploadWorkflowDataFile(file, userData, commit, client, program, mappingId, workflowId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + return result; + } + public List createTraits(int numToCreate) { List traits = new ArrayList<>(); for (int i = 0; i < numToCreate; i++) { From 38731955624881411d3f116befb8b5f1be300be5 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 20 Jun 2024 21:43:48 -0400 Subject: [PATCH 147/203] use lists instead of sets when collecting dynamic columns --- .../middleware/process/ImportTableProcess.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 5ee040dd1..a1425069c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -96,7 +96,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Collect the columns for observation variable data List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); - Set varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toSet()); + List varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toList()); // Collect the columns for observation timestamps List> timestampCols = dynamicCols.stream().filter(col -> col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); @@ -109,7 +109,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { return new ValidationErrors(); }); ValidationErrors validationErrors = context.getExpUnitContext().getValidationErrors(); - List tsValErrs = observationVariableService.validateMatchedTimestamps(varNames, timestampCols).orElse(new ArrayList<>()); + List tsValErrs = observationVariableService.validateMatchedTimestamps(Set.copyOf(varNames), timestampCols).orElse(new ArrayList<>()); for (int i = 0; i < importRows.size(); i++) { int rowNum = i; tsValErrs.forEach(validationError -> validationErrors.addError(rowNum, validationError)); @@ -129,7 +129,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Fetch the traits named in the observation variable columns Program program = context.getImportContext().getProgram(); - List traits = observationVariableService.fetchTraitsByName(varNames, program); + List traits = observationVariableService.fetchTraitsByName(Set.copyOf(varNames), program); // Map trait by phenotype column name Map traitByPhenoColName = traits.stream().collect( @@ -142,7 +142,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { ); // Sort the traits to match the order of the headers in the import file - List sortedTraits = experimentUtil.sortByField(List.copyOf(varNames), new ArrayList<>(traits), TraitEntity::getObservationVariableName); + List sortedTraits = experimentUtil.sortByField(varNames, new ArrayList<>(traits), TraitEntity::getObservationVariableName); // Get the pending observation dataset PendingImportObject pendingTrial = ExperimentUtilities.getSingleEntryValue(context.getExpUnitContext().getTrialByNameNoScope()).orElseThrow(()->new UnprocessableEntityException(MULTIPLE_EXP_TITLES.getValue())); @@ -150,9 +150,9 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { PendingImportObject pendingDataset = context.getExpUnitContext().getObsVarDatasetByName().get(datasetName); // Add new phenotypes to the pending observation dataset list (NOTE: "obsVarName [programKey]" is used instead of obsVarDbId) - // TODO: Change to using actual dbIds as per the BrAPI spec, instead of namespaced obsVar names (necessary for Breedbase) - Set datasetObsVarDbIds = pendingDataset.getBrAPIObject().getData().stream().collect(Collectors.toSet()); - Set phenoDbIds = sortedTraits.stream().map(t->Utilities.appendProgramKey(t.getObservationVariableName(), program.getKey())).collect(Collectors.toSet()); + // TODO: Change to using actual dbIds as per the BrAPI spec, instead of namespaced obsVar names (was necessary for Breedbase) + List datasetObsVarDbIds = pendingDataset.getBrAPIObject().getData().stream().collect(Collectors.toList()); + List phenoDbIds = sortedTraits.stream().map(t->Utilities.appendProgramKey(t.getObservationVariableName(), program.getKey())).collect(Collectors.toList()); phenoDbIds.removeAll(datasetObsVarDbIds); pendingDataset.getBrAPIObject().getData().addAll(phenoDbIds); From 80b8e5c97ec6e152a647eb98f766f119d0066801 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 20 Jun 2024 22:28:46 -0400 Subject: [PATCH 148/203] prevent blank values from overwriting observation data --- .../middleware/process/ImportTableProcess.java | 2 +- .../brapps/importer/ExperimentFileImportTest.java | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index a1425069c..068b130f4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -241,7 +241,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { BrAPIObservation observation = gson.fromJson(gson.toJson(observationByObsHash.get(observationHash)), BrAPIObservation.class); // Is there a change to the prior data? - if (!cellData.equals(observation.getValue()) || (cell.timestamp != null && !OffsetDateTime.parse(cell.timestamp).equals(observation.getObservationTimeStamp()))) { + if ((!cellData.isBlank() && !cellData.equals(observation.getValue())) || (cell.timestamp != null && !OffsetDateTime.parse(cell.timestamp).equals(observation.getObservationTimeStamp()))) { // Is prior data protected? boolean canOverwrite = context.getImportContext().isCommit() && "false".equals( row.getOverwrite() == null ? "false" : row.getOverwrite()); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 13e94dee9..21cde0fa6 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1156,10 +1156,12 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); + Map bothPhenotypeObservations = new HashMap<>(newObservation); + bothPhenotypeObservations.put(traits.get(0).getObservationVariableName(), "1"); if(commit) { - assertRowSaved(newObservation, program, traits); + assertRowSaved(bothPhenotypeObservations, program, traits); } else { - assertValidPreviewRow(newObservation, row, program, traits); + assertValidPreviewRow(bothPhenotypeObservations, row, program, traits); } } From ef880c2444b6b9cc45e6fc9238825d833555780a Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:21:26 -0400 Subject: [PATCH 149/203] delete vestigial classes --- .../action/create/misc/BrAPICreation.java | 94 ------------------- .../create/misc/BrAPIDatasetCreation.java | 41 -------- .../create/misc/BrAPIObservationCreation.java | 32 ------- .../misc/BrAPIObservationUnitCreation.java | 39 -------- .../create/misc/BrAPIStudyCreation.java | 37 -------- .../create/misc/BrAPITrialCreation.java | 31 ------ .../action/create/misc/LocationCreation.java | 39 -------- .../read/WorkflowReadInitialization.java | 3 +- ...rAPIDatasetReadWorkflowInitialization.java | 40 -------- ...PIGermplasmReadWorkflowInitialization.java | 38 -------- ...rvationUnitReadWorkflowInitialization.java | 40 -------- .../action/read/misc/BrAPIRead.java | 42 --------- .../misc/BrAPIReadWorkflowInitialization.java | 80 ---------------- .../BrAPIStudyReadWorkflowInitialization.java | 34 ------- .../action/read/misc/BrAPITrialRead.java | 28 ------ .../BrAPITrialReadWorkflowInitialization.java | 29 ------ .../LocationReadWorkflowInitialization.java | 42 --------- .../action/update/misc/BrAPIUpdate.java | 72 -------------- .../middleware/commit/BrAPIDatasetCommit.java | 2 - .../commit/BrAPIObservationUnitCommit.java | 2 - .../middleware/commit/BrAPIStudyCommit.java | 3 - .../middleware/commit/LocationCommit.java | 1 - .../commit/misc/BrAPIDatasetCreation.java | 57 ----------- .../misc/BrAPILocationCreationMiddleware.java | 73 -------------- .../BrAPIObservationUpdateMiddleware.java | 68 -------------- .../misc/BrAPITrialUpdateMiddleware.java | 50 ---------- .../initialize/WorkflowInitialization.java | 7 +- .../middleware/misc/RequiredBrAPIData.java | 35 ------- .../middleware/misc/RequiredDatasets.java | 71 -------------- .../middleware/misc/RequiredGermplasm.java | 67 ------------- .../middleware/misc/RequiredLocations.java | 68 -------------- .../misc/RequiredObservationUnits.java | 64 ------------- .../middleware/misc/RequiredStudies.java | 70 -------------- .../middleware/misc/RequiredTrials.java | 40 -------- .../process/misc/brapi/FieldValidation.java | 30 ------ .../misc/brapi/ImportPreviewStatistics.java | 32 ------- .../misc/brapi/NewPendingBrAPIObjects.java | 25 ----- 37 files changed, 3 insertions(+), 1523 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPICreation.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIDatasetCreation.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIObservationCreation.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIObservationUnitCreation.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIStudyCreation.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPITrialCreation.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/LocationCreation.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIDatasetReadWorkflowInitialization.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIGermplasmReadWorkflowInitialization.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIObservationUnitReadWorkflowInitialization.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIRead.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIReadWorkflowInitialization.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIStudyReadWorkflowInitialization.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialRead.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialReadWorkflowInitialization.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/LocationReadWorkflowInitialization.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/misc/BrAPIUpdate.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPIDatasetCreation.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPILocationCreationMiddleware.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPIObservationUpdateMiddleware.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPITrialUpdateMiddleware.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredBrAPIData.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredDatasets.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredGermplasm.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredLocations.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredObservationUnits.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredStudies.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredTrials.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/FieldValidation.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/ImportPreviewStatistics.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/NewPendingBrAPIObjects.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPICreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPICreation.java deleted file mode 100644 index 5a497e993..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPICreation.java +++ /dev/null @@ -1,94 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc; - -import io.micronaut.http.server.exceptions.InternalServerException; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; - -import java.util.List; -import java.util.Optional; -/** - * The BrAPICreation class is an abstract class that implements the BrAPIAction interface. It is responsible for creating entities in a BrAPI system. - * - * @param the type of entity being created - */ -@Slf4j -public abstract class BrAPICreation implements BrAPIAction { - - ExperimentImportEntity entity; - - /** - * Constructor for BrAPICreation class. - * @param context the ExpUnitMiddlewareContext object - */ - protected BrAPICreation(ExpUnitMiddlewareContext context) { - this.entity = getEntity(); - } - - protected BrAPICreation() { - } - - /** - * Abstract method to get the ExperimentImportEntity based on the ExpUnitMiddlewareContext. - * - * @return the ExperimentImportEntity object - */ - public abstract ExperimentImportEntity getEntity(); - - /** - * Executes the creation process for entities. - * @return an Optional containing the BrAPI state after execution - * @throws ApiException if an error occurs during execution - */ - public Optional execute() throws ApiException { - List newMembers = entity.copyWorkflowMembers(ImportObjectState.NEW); - try { - List createdMembers = entity.brapiPost(newMembers); - entity.updateWorkflow(createdMembers); - return Optional.of(new BrAPICreationState<>(createdMembers)); - } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { - log.error("Error creating..."); - throw new InternalServerException("Error creating...", e); - } - } - - /** - * Inner class representing the state of creation for BrAPI entities. - * @param the type of entity - */ - @Getter - public class BrAPICreationState implements BrAPIState { - - private final List members; - - /** - * Constructor for BrAPICreationState class. - * @param createdMembers the list of created members - */ - public BrAPICreationState(List createdMembers) { - this.members = createdMembers; - } - - /** - * Undo the creation operation by deleting the created members. - * @return true if undo operation is successful, false otherwise - */ - public boolean undo() { - List createdMembers = this.getMembers(); - try { - return entity.brapiDelete(createdMembers); - } catch (ApiException e) { - log.error("Error deleting..."); - throw new InternalServerException("Error deleting...", e); - } - } - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIDatasetCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIDatasetCreation.java deleted file mode 100644 index 227650c14..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIDatasetCreation.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc; - -import io.micronaut.context.ApplicationContext; -import lombok.extern.slf4j.Slf4j; -import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingDataset; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; - -@Slf4j -public class BrAPIDatasetCreation extends BrAPICreation { - ExpUnitMiddlewareContext context; - /** - * Constructor for BrAPICreation class. - * - * @param context the ExpUnitMiddlewareContext object - */ - public BrAPIDatasetCreation(ExpUnitMiddlewareContext context) { - - this.context = context; - } - - /** - * Abstract method to get the ExperimentImportEntity based on the ExpUnitMiddlewareContext. - * - * @return the ExperimentImportEntity object - */ - @Override - public ExperimentImportEntity getEntity() { - try (ApplicationContext appContext = ApplicationContext.run()) { - BrAPIListDAO brAPIListDAO = appContext.getBean(BrAPIListDAO.class); - DatasetService datasetService = appContext.getBean(DatasetService.class); - ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); - - return new PendingDataset(context, brAPIListDAO, datasetService, experimentUtilities); - } - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIObservationCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIObservationCreation.java deleted file mode 100644 index 45cb24254..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIObservationCreation.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc; - -import io.micronaut.context.ApplicationContext; -import lombok.extern.slf4j.Slf4j; -import org.brapi.v2.model.pheno.BrAPIObservation; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingObservation; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.services.OntologyService; - -@Slf4j -public class BrAPIObservationCreation extends BrAPICreation { - - ExpUnitMiddlewareContext context; - public BrAPIObservationCreation(ExpUnitMiddlewareContext context) { - - this.context = context; - } - - @Override - public ExperimentImportEntity getEntity() { - try (ApplicationContext appContext = ApplicationContext.run()) { - BrAPIObservationDAO brAPIObservationDAO = appContext.getBean(BrAPIObservationDAO.class); - OntologyService ontologyService = appContext.getBean(OntologyService.class); - ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); - - return new PendingObservation(context, brAPIObservationDAO, ontologyService, experimentUtilities); - } - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIObservationUnitCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIObservationUnitCreation.java deleted file mode 100644 index 3f284040c..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIObservationUnitCreation.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc; - -import io.micronaut.context.ApplicationContext; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingObservationUnit; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; - -public class BrAPIObservationUnitCreation extends BrAPICreation { - ExpUnitMiddlewareContext context; - /** - * Constructor for BrAPICreation class. - * - * @param context the ExpUnitMiddlewareContext object - */ - public BrAPIObservationUnitCreation(ExpUnitMiddlewareContext context) { - - this.context = context; - } - - /** - * Abstract method to get the ExperimentImportEntity based on the ExpUnitMiddlewareContext. - * - * @return the ExperimentImportEntity object - */ - @Override - public ExperimentImportEntity getEntity() { - try (ApplicationContext appContext = ApplicationContext.run()) { - BrAPIObservationUnitDAO observationUnitDAO = appContext.getBean(BrAPIObservationUnitDAO.class); - ObservationUnitService observationUnitService = appContext.getBean(ObservationUnitService.class); - ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); - - return new PendingObservationUnit(context, observationUnitDAO, observationUnitService, experimentUtilities); - } - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIStudyCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIStudyCreation.java deleted file mode 100644 index 69062693c..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPIStudyCreation.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc; - -import io.micronaut.context.ApplicationContext; -import lombok.extern.slf4j.Slf4j; -import org.brapi.v2.model.core.BrAPIStudy; -import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingStudy; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; - -@Slf4j -public class BrAPIStudyCreation extends BrAPICreation { - - ExpUnitMiddlewareContext context; - public BrAPIStudyCreation(ExpUnitMiddlewareContext context) { - - this.context = context; - } - - /** - * Abstract method to get the ExperimentImportEntity based on the ExpUnitMiddlewareContext. - * - * @return the ExperimentImportEntity object - */ - @Override - public ExperimentImportEntity getEntity() { - try (ApplicationContext appContext = ApplicationContext.run()) { - StudyService studyService = appContext.getBean(StudyService.class); - BrAPIStudyDAO brAPIStudyDAO = appContext.getBean(BrAPIStudyDAO.class); - ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); - - return new PendingStudy(context, studyService, brAPIStudyDAO, experimentUtilities); - } - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPITrialCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPITrialCreation.java deleted file mode 100644 index 721bdd3cb..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/BrAPITrialCreation.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc; - -import lombok.extern.slf4j.Slf4j; -import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; - -@Slf4j -public class BrAPITrialCreation extends BrAPICreation { - ExpUnitMiddlewareContext context; - private PendingEntityFactory pendingEntityFactory; - public BrAPITrialCreation(ExpUnitMiddlewareContext context, - PendingEntityFactory pendingEntityFactory) { - - this.context = context; - this.pendingEntityFactory = pendingEntityFactory; - } - - @Override - public ExperimentImportEntity getEntity() { -// try (ApplicationContext appContext = ApplicationContext.run()) { -// TrialService trialService = appContext.getBean(TrialService.class); -// BrAPITrialDAO brAPITrialDAO = appContext.getBean(BrAPITrialDAO.class); -// ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); -// -// return new PendingTrial(context, trialService, brAPITrialDAO, experimentUtilities); -// } - return pendingEntityFactory.pendingTrialBean(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/LocationCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/LocationCreation.java deleted file mode 100644 index 4744ac7c1..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/misc/LocationCreation.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc; - -import io.micronaut.context.ApplicationContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingLocation; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; -import org.breedinginsight.model.ProgramLocation; -import org.breedinginsight.services.ProgramLocationService; - -public class LocationCreation extends BrAPICreation { - ExpUnitMiddlewareContext context; - /** - * Constructor for BrAPICreation class. - * - * @param context the ExpUnitMiddlewareContext object - */ - public LocationCreation(ExpUnitMiddlewareContext context) { - - this.context = context; - } - - /** - * Abstract method to get the ExperimentImportEntity based on the ExpUnitMiddlewareContext. - * - * @return the ExperimentImportEntity object - */ - @Override - public ExperimentImportEntity getEntity() { - try (ApplicationContext appContext = ApplicationContext.run()) { - ProgramLocationService programLocationService = appContext.getBean(ProgramLocationService.class); - LocationService locationService = appContext.getBean(LocationService.class); - ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); - - return new PendingLocation(context, programLocationService, locationService, experimentUtilities); - } - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/WorkflowReadInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/WorkflowReadInitialization.java index 7310bd1b5..5faaaadc3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/WorkflowReadInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/WorkflowReadInitialization.java @@ -6,7 +6,6 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIReadWorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.utilities.Utilities; @@ -33,7 +32,7 @@ public Optional execute() throws ApiException { try { List fetchedMembers = entity.brapiRead(); entity.initializeWorkflow(fetchedMembers); - return Optional.of(new BrAPIReadWorkflowInitialization.BrAPIReadState(fetchedMembers)); + return Optional.of(new WorkflowReadInitialization.BrAPIReadState(fetchedMembers)); } catch(ApiException e) { log.error(String.format("Error fetching %s: %s", entity.getClass().getName(), Utilities.generateApiExceptionLogMessage(e)), e); throw new ApiException(e); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIDatasetReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIDatasetReadWorkflowInitialization.java deleted file mode 100644 index 6740135d2..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIDatasetReadWorkflowInitialization.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; - -import io.micronaut.context.ApplicationContext; -import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIReadWorkflowInitialization; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingDataset; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; - -public class BrAPIDatasetReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { - /** - * Constructs a new BrAPIReadWorkflowInitialization object with the given ExpUnitMiddlewareContext. - * Initializes the entity based on the provided context. - * - * @param context the ExpUnitMiddlewareContext used for initialization. - */ - ExpUnitMiddlewareContext context; - public BrAPIDatasetReadWorkflowInitialization(ExpUnitMiddlewareContext context) { - this.context = context; - } - - /** - * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. - * - * @return The ExperimentImportEntity representing the BrAPI entity being acted on. - */ - @Override - public ExperimentImportEntity getEntity() { - try (ApplicationContext appContext = ApplicationContext.run()) { - BrAPIListDAO brAPIListDAO = appContext.getBean(BrAPIListDAO.class); - DatasetService datasetService = appContext.getBean(DatasetService.class); - ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); - - return new PendingDataset(context, brAPIListDAO, datasetService, experimentUtilities); - } - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIGermplasmReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIGermplasmReadWorkflowInitialization.java deleted file mode 100644 index afa840b53..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIGermplasmReadWorkflowInitialization.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; - -import io.micronaut.context.ApplicationContext; -import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIReadWorkflowInitialization; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingGermplasm; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.GermplasmService; - -public class BrAPIGermplasmReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { - ExpUnitMiddlewareContext context; - /** - * Constructs a new BrAPIReadWorkflowInitialization object with the given ExpUnitMiddlewareContext. - * Initializes the entity based on the provided context. - * - * @param context the ExpUnitMiddlewareContext used for initialization. - */ - public BrAPIGermplasmReadWorkflowInitialization(ExpUnitMiddlewareContext context) { - this.context = context; - } - - /** - * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. - * - * @return The ExperimentImportEntity representing the BrAPI entity being acted on. - */ - @Override - public ExperimentImportEntity getEntity() { - try (ApplicationContext appContext = ApplicationContext.run()) { - GermplasmService germplasmService = appContext.getBean(GermplasmService.class); - ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); - - return new PendingGermplasm(context, germplasmService, experimentUtilities); - } - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIObservationUnitReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIObservationUnitReadWorkflowInitialization.java deleted file mode 100644 index 7e6c7a21a..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIObservationUnitReadWorkflowInitialization.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; - -import io.micronaut.context.ApplicationContext; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIReadWorkflowInitialization; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingObservationUnit; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; - -public class BrAPIObservationUnitReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { - ExpUnitMiddlewareContext context; - /** - * Constructs a new BrAPIReadWorkflowInitialization object with the given ExpUnitMiddlewareContext. - * Initializes the entity based on the provided context. - * - * @param context the ExpUnitMiddlewareContext used for initialization. - */ - public BrAPIObservationUnitReadWorkflowInitialization(ExpUnitMiddlewareContext context) { - this.context = context; - } - - /** - * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. - * - * @return The ExperimentImportEntity representing the BrAPI entity being acted on. - */ - @Override - public ExperimentImportEntity getEntity() { - try (ApplicationContext appContext = ApplicationContext.run()) { - BrAPIObservationUnitDAO observationUnitDAO = appContext.getBean(BrAPIObservationUnitDAO.class); - ObservationUnitService observationUnitService = appContext.getBean(ObservationUnitService.class); - ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); - - return new PendingObservationUnit(context, observationUnitDAO, observationUnitService, experimentUtilities); - } - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIRead.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIRead.java deleted file mode 100644 index 82201dea4..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIRead.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; - -import io.micronaut.http.server.exceptions.InternalServerException; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; - -import java.util.List; -import java.util.Optional; - -@Slf4j -public abstract class BrAPIRead implements BrAPIAction { - ExperimentImportEntity entity; - - protected BrAPIRead(ExpUnitMiddlewareContext context) { - this.entity = getEntity(); - } - - protected BrAPIRead() { - } - - public Optional execute() throws ApiException { - try { - List fetchedMembers = entity.brapiRead(); - return Optional.of(new BrAPIReadState(fetchedMembers)); - } catch(ApiException e) { - // TODO: add specific error messages to entity service - log.error("Error reading..."); - throw new InternalServerException("Error reading...", e); - } - } - @Getter - public static class BrAPIReadState implements BrAPIState { - - private final List members; - public BrAPIReadState(List fetchedMembers) { this.members = fetchedMembers; } - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIReadWorkflowInitialization.java deleted file mode 100644 index f19e8ceb3..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIReadWorkflowInitialization.java +++ /dev/null @@ -1,80 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; - -import io.micronaut.http.server.exceptions.InternalServerException; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.utilities.Utilities; - -import javax.inject.Inject; -import java.util.List; -import java.util.Optional; - -/** - * This abstract class, BrAPIReadWorkflowInitialization, is responsible for initializing and executing a read workflow for a given BrAPI entity. - * It implements the BrAPIAction interface and uses the ExperimentImportEntity to perform read operations. - * - * @param the type of entity for which the workflow is initialized - */ -@Slf4j -public abstract class BrAPIReadWorkflowInitialization implements BrAPIAction { - - ExperimentImportEntity entity; // The entity used for read operations initialization - PendingEntityFactory pendingEntityFactory; - - /** - * Constructs a new BrAPIReadWorkflowInitialization object with the given ExpUnitMiddlewareContext. - * Initializes the entity based on the provided context. - * - * @param context the ExpUnitMiddlewareContext used for initialization. - */ - @Inject - protected BrAPIReadWorkflowInitialization(PendingEntityFactory pendingEntityFactory) { - this.entity = getEntity(); - this.pendingEntityFactory = pendingEntityFactory; - } - - @Inject - protected BrAPIReadWorkflowInitialization() { - } - - /** - * Executes the read workflow by fetching members from the entity and initializing the workflow. - * - * @return an Optional containing the BrAPIState representing the completed read workflow - * @throws ApiException if an error occurs during execution - */ - public Optional execute() throws ApiException { - try { - List fetchedMembers = entity.brapiRead(); - entity.initializeWorkflow(fetchedMembers); - return Optional.of(new BrAPIReadState(fetchedMembers)); - } catch(ApiException e) { - log.error(String.format("Error fetching %s: %s", entity.getClass().getName(), Utilities.generateApiExceptionLogMessage(e)), e); - throw new ApiException(e); - } - } - - /** - * The state class representing the result of a read operation. - * - * @param the type of entity members contained in the state - */ - @Getter - public static class BrAPIReadState implements BrAPIState { - - private final List members; // The list of members fetched during the read operation - - /** - * Constructs a new BrAPIReadState object with the provided list of members. - * - * @param fetchedMembers the list of members fetched during the read operation - */ - public BrAPIReadState(List fetchedMembers) { this.members = fetchedMembers; } - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIStudyReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIStudyReadWorkflowInitialization.java deleted file mode 100644 index beb01b3de..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPIStudyReadWorkflowInitialization.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; - -import io.micronaut.context.ApplicationContext; -import org.brapi.v2.model.core.BrAPIStudy; -import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIReadWorkflowInitialization; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingStudy; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; - -public class BrAPIStudyReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { - ExpUnitMiddlewareContext context; - public BrAPIStudyReadWorkflowInitialization(ExpUnitMiddlewareContext context) { - this.context = context; - } - - /** - * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. - * - * @return The ExperimentImportEntity representing the BrAPI entity being acted on. - */ - @Override - public ExperimentImportEntity getEntity() { - try (ApplicationContext appContext = ApplicationContext.run()) { - StudyService studyService = appContext.getBean(StudyService.class); - BrAPIStudyDAO brAPIStudyDAO = appContext.getBean(BrAPIStudyDAO.class); - ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); - - return new PendingStudy(context, studyService, brAPIStudyDAO, experimentUtilities); - } - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialRead.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialRead.java deleted file mode 100644 index af136d465..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialRead.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; - -import lombok.extern.slf4j.Slf4j; -import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; - -@Slf4j -public class BrAPITrialRead extends BrAPIRead { - - ExpUnitMiddlewareContext context; - public BrAPITrialRead(ExpUnitMiddlewareContext context) { - - this.context = context; - } - - @Override - public ExperimentImportEntity getEntity() { -// try (ApplicationContext appContext = ApplicationContext.run()) { -// TrialService trialService = appContext.getBean(TrialService.class); -// BrAPITrialDAO brAPITrialDAO = appContext.getBean(BrAPITrialDAO.class); -// ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); -// -// return new PendingTrial(context, trialService, brAPITrialDAO, experimentUtilities); -// } - return null; - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialReadWorkflowInitialization.java deleted file mode 100644 index a36ab85d2..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/BrAPITrialReadWorkflowInitialization.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; - -import lombok.extern.slf4j.Slf4j; -import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; - -@Slf4j -public class BrAPITrialReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { - - ExpUnitMiddlewareContext context; - - public BrAPITrialReadWorkflowInitialization(ExpUnitMiddlewareContext context) { - this.context = context; - } - - @Override - public ExperimentImportEntity getEntity() { - - return pendingEntityFactory.pendingTrialBean(context); -// try (ApplicationContext appContext = ApplicationContext.run()) { -// TrialService trialService = appContext.getBean(TrialService.class); -// BrAPITrialDAO brAPITrialDAO = appContext.getBean(BrAPITrialDAO.class); -// ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); -// -// return new PendingTrial(context, trialService, brAPITrialDAO, experimentUtilities); -// } - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/LocationReadWorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/LocationReadWorkflowInitialization.java deleted file mode 100644 index 0d8285703..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/misc/LocationReadWorkflowInitialization.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc; - -import io.micronaut.context.ApplicationContext; -import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIReadWorkflowInitialization; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingLocation; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; -import org.breedinginsight.model.ProgramLocation; -import org.breedinginsight.services.ProgramLocationService; - -@Slf4j -public class LocationReadWorkflowInitialization extends BrAPIReadWorkflowInitialization { - ExpUnitMiddlewareContext context; - /** - * Constructs a new BrAPIReadWorkflowInitialization object with the given ExpUnitMiddlewareContext. - * Initializes the entity based on the provided context. - * - * @param context the ExpUnitMiddlewareContext used for initialization. - */ - public LocationReadWorkflowInitialization(ExpUnitMiddlewareContext context) { - this.context = context; - } - - /** - * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. - * - * @return The ExperimentImportEntity representing the BrAPI entity being acted on. - */ - @Override - public ExperimentImportEntity getEntity() { - try (ApplicationContext appContext = ApplicationContext.run()) { - ProgramLocationService programLocationService = appContext.getBean(ProgramLocationService.class); - LocationService locationService = appContext.getBean(LocationService.class); - ExperimentUtilities experimentUtilities = appContext.getBean(ExperimentUtilities.class); - - return new PendingLocation(context, programLocationService, locationService, experimentUtilities); - } - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/misc/BrAPIUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/misc/BrAPIUpdate.java deleted file mode 100644 index c7e2c534a..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/misc/BrAPIUpdate.java +++ /dev/null @@ -1,72 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.misc; - -import io.micronaut.http.server.exceptions.InternalServerException; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; - -import java.util.List; -import java.util.Optional; - -@Slf4j -public abstract class BrAPIUpdate implements BrAPIAction { - ExperimentImportEntity entity; - PendingEntityFactory pendingEntityFactory; - - protected BrAPIUpdate(PendingEntityFactory pendingEntityFactory) { - - this.entity = getEntity(); - this.pendingEntityFactory = pendingEntityFactory; - } - - protected BrAPIUpdate() { - } - - public Optional execute() throws ApiException { - return saveAndUpdateCache(entity.copyWorkflowMembers(ImportObjectState.MUTATED)); - } - - public Optional> getBrAPIState() { - try { - return Optional.of(new BrAPIUpdateState(entity.getBrAPIState(ImportObjectState.MUTATED))); - } catch (ApiException e) { - // TODO: add specific error messages to entity service - log.error("Error getting..."); - throw new InternalServerException("Error getting...", e); - } - - } - - protected Optional saveAndUpdateCache(List members) { - return Optional.ofNullable(members).map(changes -> { - try { - List savedMembers = entity.brapiPut(changes); - // TODO: set updated fields of workflow brapi object - // entity.updateWorkflow(savedMembers); - return new BrAPIUpdateState(savedMembers); - } catch (ApiException e) { - // TODO: add specific error messages to entity service - log.error("Error updating..."); - throw new InternalServerException("Error updating...", e); - } - }); - } - - @Getter - public class BrAPIUpdateState implements BrAPIState { - private final List members; - - public BrAPIUpdateState(List existingMembers) { this.members = existingMembers; } - - public boolean restore() { - return saveAndUpdateCache(this.getMembers()).isPresent(); - } - } - -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index 36f87f4a5..75c0b4671 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -6,8 +6,6 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPIDatasetCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIUpdateFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index 48493dbbe..64f700f22 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -6,8 +6,6 @@ import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPIObservationUnitCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java index ad2819877..d4a08738e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -1,14 +1,11 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; import io.micronaut.context.annotation.Prototype; -import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPIStudyCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index 3c46fafb7..8857f5445 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -5,7 +5,6 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.misc.BrAPICreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPIDatasetCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPIDatasetCreation.java deleted file mode 100644 index ca66ccdce..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPIDatasetCreation.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.misc; - -import io.micronaut.context.annotation.Prototype; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPIListSummary; -import org.brapi.v2.model.core.request.BrAPIListNewRequest; -import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; - -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -@Slf4j -@Prototype -public class BrAPIDatasetCreation extends ExpUnitMiddleware { - - ExperimentUtilities experimentUtilities; - BrAPIListDAO brapiListDAO; - private List newDatasetRequests; - @Inject - public BrAPIDatasetCreation(ExperimentUtilities experimentUtilities, BrAPIListDAO brapiListDAO) { - this.experimentUtilities = experimentUtilities; - this.brapiListDAO = brapiListDAO; - } - @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { - // Construct request - newDatasetRequests = experimentUtilities.getNewObjects(context.getPendingData().getObsVarDatasetByName(), BrAPIListDetails.class).stream().map(details -> { - BrAPIListNewRequest request = new BrAPIListNewRequest(); - request.setListName(details.getListName()); - request.setListType(details.getListType()); - request.setExternalReferences(details.getExternalReferences()); - request.setAdditionalInfo(details.getAdditionalInfo()); - request.data(details.getData()); - return request; - }).collect(Collectors.toList()); - - List createdDatasets = null; - try { - // Create entities in brapi service - createdDatasets = new ArrayList<>(brapiListDAO.createBrAPILists(newDatasetRequests, context.getImportContext().getProgram().getId(), context.getImportContext().getUpload())); - - // Update the context cache by setting the system-generated dbId for each newly created dataset - createdDatasets.forEach(summary -> context.getPendingData().getObsVarDatasetByName().get(summary.getListName()).getBrAPIObject().setListDbId(summary.getListDbId())); - } catch (ApiException e) { - throw new RuntimeException(e); - } - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPILocationCreationMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPILocationCreationMiddleware.java deleted file mode 100644 index 882635d8e..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPILocationCreationMiddleware.java +++ /dev/null @@ -1,73 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.misc; - -import io.micronaut.context.annotation.Prototype; -import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.api.auth.AuthenticatedUser; -import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.model.ProgramLocation; -import org.breedinginsight.services.ProgramLocationService; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; - -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -@Slf4j -@Prototype -public class BrAPILocationCreationMiddleware extends ExpUnitMiddleware { - - ExperimentUtilities experimentUtilities; - ProgramLocationService programLocationService; - private List newLocations; - @Inject - public BrAPILocationCreationMiddleware(ExperimentUtilities experimentUtilities, ProgramLocationService programLocationService) { - this.experimentUtilities = experimentUtilities; - this.programLocationService = programLocationService; - } - @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { - // Construct request - newLocations = experimentUtilities.getNewObjects(context.getPendingData().getLocationByName(), ProgramLocation.class) - .stream() - .map(location -> ProgramLocationRequest.builder() - .name(location.getName()) - .build()) - .collect(Collectors.toList()); - - // Create acting user - AuthenticatedUser actingUser = new AuthenticatedUser(context.getImportContext().getUpload().getUpdatedByUser().getName(), new ArrayList<>(), context.getImportContext().getUpload().getUpdatedByUser().getId(), new ArrayList<>()); - - // Create new locations in brapi service - List createdLocations = null; - try { - createdLocations = new ArrayList<>(programLocationService.create(actingUser, context.getImportContext().getProgram().getId(), newLocations)); - - // Update the context cache - for (ProgramLocation createdLocation : createdLocations) { - - // Set the system-generated dbId for each newly created location - context.getPendingData().getLocationByName().get(createdLocation.getName()).getBrAPIObject().setLocationDbId(createdLocation.getLocationDbId()); - - // Set the location dbid for cached studies - context.getPendingData().getStudyByNameNoScope().values().stream() - .filter(study -> createdLocation.getId().toString().equals(study.getBrAPIObject().getLocationDbId())) - .forEach(study -> study.getBrAPIObject().setLocationDbId(createdLocation.getLocationDbId())); - } - - } catch (MissingRequiredInfoException e) { - throw new RuntimeException(e); - } catch (UnprocessableEntityException e) { - throw new RuntimeException(e); - } catch (DoesNotExistException e) { - throw new RuntimeException(e); - } - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPIObservationUpdateMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPIObservationUpdateMiddleware.java deleted file mode 100644 index 3463af873..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPIObservationUpdateMiddleware.java +++ /dev/null @@ -1,68 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.misc; - -import io.micronaut.context.annotation.Prototype; -import io.micronaut.http.server.exceptions.InternalServerException; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.pheno.BrAPIObservation; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.utilities.Utilities; - -import javax.inject.Inject; -import java.util.Map; -import java.util.Objects; - -@Slf4j -@Prototype -public class BrAPIObservationUpdateMiddleware extends ExpUnitMiddleware { - - ExperimentUtilities experimentUtilities; - BrAPIObservationDAO brapiObservationDAO; - private Map mutatedObservationByDbId; - - @Inject - public BrAPIObservationUpdateMiddleware(ExperimentUtilities experimentUtilities, BrAPIObservationDAO brapiObservationDAO) { - this.experimentUtilities = experimentUtilities; - this.brapiObservationDAO = brapiObservationDAO; - } - @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { - - mutatedObservationByDbId = experimentUtilities.getMutationsByObjectId(context.getPendingData().getPendingObservationByHash(), BrAPIObservation::getObservationDbId, BrAPIObservation.class); - - mutatedObservationByDbId.forEach((id, observation) -> { - try { - if (observation == null) { - throw new Exception("Null observation"); - } - BrAPIObservation updatedObs = brapiObservationDAO.updateBrAPIObservation(id, observation, context.getImportContext().getProgram().getId()); - - if (updatedObs == null) { - throw new Exception("Null updated observation"); - } - - if (!Objects.equals(observation.getValue(), updatedObs.getValue()) - || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { - String message; - if(!Objects.equals(observation.getValue(), updatedObs.getValue())) { - message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); - } else { - message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); - } - throw new Exception(message); - } - } catch (ApiException e) { - log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException("Error saving experiment import", e); - } catch (Exception e) { - log.error("Error updating observation: ", e); - throw new InternalServerException(e.getMessage(), e); - } - }); - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPITrialUpdateMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPITrialUpdateMiddleware.java deleted file mode 100644 index cdea7f417..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/misc/BrAPITrialUpdateMiddleware.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.misc; - -import io.micronaut.context.annotation.Prototype; -import io.micronaut.http.server.exceptions.InternalServerException; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.utilities.Utilities; - -import javax.inject.Inject; -import java.util.Map; - -@Slf4j -@Prototype -public class BrAPITrialUpdateMiddleware extends ExpUnitMiddleware { - - ExperimentUtilities experimentUtilities; - BrAPITrialDAO brapiTrialDAO; - private Map mutatedTrialsById; - - @Inject - public BrAPITrialUpdateMiddleware(ExperimentUtilities experimentUtilities, BrAPITrialDAO brapiTrialDAO) { - this.experimentUtilities = experimentUtilities; - this.brapiTrialDAO = brapiTrialDAO; - } - @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { - // Construct request - mutatedTrialsById = experimentUtilities.getMutationsByObjectId(context.getPendingData().getTrialByNameNoScope(), BrAPITrial::getTrialDbId, BrAPITrial.class); - - mutatedTrialsById.forEach((id, trial) -> { - try { - // Update entities in the brapi service - brapiTrialDAO.updateBrAPITrial(id, trial, context.getImportContext().getProgram().getId()); - } catch (ApiException e) { - log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException("Error saving experiment import", e); - } catch (Exception e) { - log.error("Error updating dataset observation variables: ", e); - throw new InternalServerException(e.getMessage(), e); - } - }); - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index b6e1b03df..012551c63 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -8,11 +8,8 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.*; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIStudyReadWorkflowInitialization; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIDatasetReadWorkflowInitialization; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.misc.BrAPIGermplasmReadWorkflowInitialization; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.BrAPIReadFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.WorkflowReadInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredBrAPIData.java deleted file mode 100644 index f4ae66dde..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredBrAPIData.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.misc; - -import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; - -import javax.inject.Inject; - -@Slf4j -public class RequiredBrAPIData extends ExpUnitMiddleware { - ExpUnitMiddleware middleware; - - @Inject - public RequiredBrAPIData(RequiredObservationUnits requiredObservationUnits, - RequiredTrials requiredTrials, - RequiredStudies requiredStudies, - RequiredLocations requiredLocations, - RequiredDatasets requiredDatasets, - RequiredGermplasm requiredGermplasm) { - - this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( - requiredObservationUnits, // Fetch the BrAPI units for the required exp unit ids - requiredTrials, // Fetch the BrAPI trials belonging to the exp units - requiredStudies, // Fetch the BrAPI studies belonging to the exp units - requiredLocations, // Fetch the BrAPI locations belonging to the exp units - requiredDatasets, // Fetch the dataset belonging to the exp units - requiredGermplasm); // Fetch the germplasm belonging to the exp units - } - - @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { - log.debug("reading required BrAPI data from BrAPI service"); - return this.middleware.process(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredDatasets.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredDatasets.java deleted file mode 100644 index 368086c7a..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredDatasets.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.misc; - -import io.micronaut.context.annotation.Prototype; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; -import org.breedinginsight.model.Program; - -import javax.inject.Inject; -import java.util.HashMap; -import java.util.Map; - -@Slf4j -@Prototype -public class RequiredDatasets extends ExpUnitMiddleware { - private final DatasetService datasetService; - - @Inject - public RequiredDatasets(DatasetService datasetService) { - this.datasetService = datasetService; - } - - @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { - Program program; - String datasetId; - BrAPIListDetails dataset = null; - PendingImportObject pendingDataset; - Map> pendingTrialByNameNoScope; - Map> pendingDatasetByName; - - program = context.getImportContext().getProgram(); - pendingTrialByNameNoScope = context.getPendingData().getTrialByNameNoScope(); - - // nothing to do if there are no trials with dataset ids - if (pendingTrialByNameNoScope.size() == 0 || - !pendingTrialByNameNoScope.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { - return processNext(context); - } - log.debug("fetching from BrAPI service, datasets belonging to required units"); - - // Get the id of the dataset belonging to the required exp units - datasetId = pendingTrialByNameNoScope.values().iterator().next().getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) - .getAsString(); - try { - // Get the dataset belonging to required exp units - dataset = datasetService.fetchDatasetById(datasetId, program).orElseThrow(ApiException::new); - - // Construct the pending dataset from the BrAPI observation variable list - pendingDataset = datasetService.constructPIOFromDataset(dataset, program); - - // Construct a hashmap to look up the pending dataset by dataset name - pendingDatasetByName = new HashMap<>(); - pendingDatasetByName.put(dataset.getListName(), pendingDataset); - - // Add the map to the context for use in processing import - context.getPendingData().setObsVarDatasetByName(pendingDatasetByName); - } catch (ApiException e) { - this.compensate(context); - } - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredGermplasm.java deleted file mode 100644 index 92b92c04f..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredGermplasm.java +++ /dev/null @@ -1,67 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.misc; - -import io.micronaut.context.annotation.Prototype; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.GermplasmService; -import org.breedinginsight.model.Program; - -import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; - -@Slf4j -@Prototype -public class RequiredGermplasm extends ExpUnitMiddleware { - private final GermplasmService germplasmService; - - @Inject - public RequiredGermplasm(GermplasmService germplasmService) { - - this.germplasmService = germplasmService; - } - - @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { - Program program; - Set germplasmDbIds; - List brapiGermplasm = null; - List> pendingGermplasm; - Map> pendingGermplasmByGID; - Map> pendingUnitByNameNoScope; - - program = context.getImportContext().getProgram(); - pendingUnitByNameNoScope = context.getPendingData().getObservationUnitByNameNoScope(); - - // nothing to do if there are no observation units - if (pendingUnitByNameNoScope.size() == 0) { - return processNext(context); - } - log.debug("fetching from BrAPI service, germplasm belonging to required units"); - - // Get the dbIds of the germplasm belonging to the required exp units - germplasmDbIds = pendingUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); - try { - // Get the dataset belonging to required exp units - brapiGermplasm = germplasmService.fetchGermplasmByDbId(new HashSet<>(germplasmDbIds), program); - - // Construct the pending germplasm from the BrAPI locations - pendingGermplasm = brapiGermplasm.stream().map(germplasmService::constructPIOFromBrapiGermplasm).collect(Collectors.toList()); - - // Construct a hashmap to look up the pending germplasm by gid - pendingGermplasmByGID = pendingGermplasm.stream().collect(Collectors.toMap(germplasmService::getGIDFromGermplasmPIO, pio -> pio)); - - // Add the map to the context for use in processing import - context.getPendingData().setExistingGermplasmByGID(pendingGermplasmByGID); - } catch (ApiException e) { - this.compensate(context); - } - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredLocations.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredLocations.java deleted file mode 100644 index ae7526a08..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredLocations.java +++ /dev/null @@ -1,68 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.misc; - -import io.micronaut.context.annotation.Prototype; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPIStudy; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.ProgramLocation; - -import javax.inject.Inject; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -@Slf4j -@Prototype -public class RequiredLocations extends ExpUnitMiddleware { - LocationService locationService; - - @Inject - public RequiredLocations(LocationService locationService) { - this.locationService = locationService; - } - - @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { - Program program; - Set locationDbIds; - List brapiLocations; - List> pendingLocations; - Map> pendingStudyByNameNoScope; - Map> pendingLocationByName; - - program = context.getImportContext().getProgram(); - pendingStudyByNameNoScope = context.getPendingData().getStudyByNameNoScope(); - - // nothing to do if there are no required units - if (pendingStudyByNameNoScope.size() == 0) { - return processNext(context); - } - log.debug("fetching from BrAPI service, locations belonging to required units"); - - // Get the dbIds of the studies belonging to the required exp units - locationDbIds = pendingStudyByNameNoScope.values().stream().map(pio -> pio.getBrAPIObject().getLocationDbId()).collect(Collectors.toSet()); - try { - // Get the locations belonging to required exp units - brapiLocations = locationService.fetchLocationsByDbId(locationDbIds, program); - - // Construct the pending locations from the BrAPI locations - pendingLocations = brapiLocations.stream().map(locationService::constructPIOFromBrapiLocation).collect(Collectors.toList()); - - // Construct a hashmap to look up the pending location by location name - pendingLocationByName = pendingLocations.stream().collect(Collectors.toMap(pio -> pio.getBrAPIObject().getName(), pio -> pio)); - - // Add the map to the context for use in processing import - context.getPendingData().setLocationByName(pendingLocationByName); - } catch (ApiException e) { - this.compensate(context); - } - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredObservationUnits.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredObservationUnits.java deleted file mode 100644 index 101c27c1b..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredObservationUnits.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.misc; - -import io.micronaut.context.annotation.Prototype; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; -import org.breedinginsight.model.Program; - -import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; - -@Slf4j -@Prototype -public class RequiredObservationUnits extends ExpUnitMiddleware { - ObservationUnitService observationUnitService; - - @Inject - public RequiredObservationUnits(ObservationUnitService observationUnitService) { - this.observationUnitService = observationUnitService; - } - - @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { - Program program; - Set expUnitIds; - List missingIds; - List brapiUnits; - List> pendingUnits; - Map> pendingUnitById; - Map> pendingUnitByNameNoScope; - - log.debug("fetching required exp units from BrAPI service"); - program = context.getImportContext().getProgram(); - - // Collect deltabreed-generated exp unit ids listed in the import - expUnitIds = context.getExpUnitContext().getReferenceOUIds(); - try { - // For each id fetch the observation unit from the brapi data store - brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); - - // Construct pending import objects from the units - pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); - - // Construct a hashmap to look up the pending unit by ID - pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); - - // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed - pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); - - // add maps to the context for use in processing import - context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); - context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); - } catch (ApiException e) { - this.compensate(context); - } - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredStudies.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredStudies.java deleted file mode 100644 index 741b625fd..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredStudies.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.misc; - -import io.micronaut.context.annotation.Prototype; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; -import org.breedinginsight.model.Program; -import org.breedinginsight.utilities.Utilities; - -import javax.inject.Inject; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -@Slf4j -@Prototype -public class RequiredStudies extends ExpUnitMiddleware { - StudyService studyService; - - @Inject - public RequiredStudies(StudyService studyService) { - this.studyService = studyService; - } - - @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { - Program program; - Map> pendingUnitByNameNoScope; - Set studyDbIds; - List brAPIStudies; - List> pendingStudies; - Map> pendingStudyByNameNoScope; - - program = context.getImportContext().getProgram(); - pendingUnitByNameNoScope = context.getPendingData().getObservationUnitByNameNoScope(); - - // nothing to do if there are no required units - if (pendingUnitByNameNoScope.size() == 0) { - return processNext(context); - } - log.debug("fetching from BrAPI service studies belonging to required units"); - - // Get the dbIds of the studies belonging to the required exp units - studyDbIds = pendingUnitByNameNoScope.values().stream().map(studyService::getStudyDbIdBelongingToPendingUnit).collect(Collectors.toSet()); - - try { - // Get the BrAPI studies belonging to required exp units - brAPIStudies = studyService.fetchBrapiStudiesByDbId(studyDbIds, program); - - // Construct the pending studies from the BrAPI trials - pendingStudies = brAPIStudies.stream().map(pio -> studyService.constructPIOFromBrapiStudy(pio, program)).collect(Collectors.toList()); - - // Construct a hashmap to look up the pending study by study name with the program key removed - pendingStudyByNameNoScope = pendingStudies.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKeyAndUnknownAdditionalData(pio.getBrAPIObject().getStudyName(), program.getKey()), pio -> pio)); - - // Add the map to the context for use in processing import - context.getPendingData().setStudyByNameNoScope(pendingStudyByNameNoScope); - } catch (ApiException e) { - this.compensate(context); - } - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredTrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredTrials.java deleted file mode 100644 index a7da97d12..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/misc/RequiredTrials.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.misc; - -import io.micronaut.context.annotation.Prototype; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; - -import javax.inject.Inject; - -@Slf4j -@Prototype -public class RequiredTrials extends ExpUnitMiddleware { - TrialService trialService; - - @Inject - public RequiredTrials(TrialService trialService) { - this.trialService = trialService; - } - - @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { - // Nothing to do if there are no required units - if (context.getPendingData().getObservationUnitByNameNoScope().size() == 0) { - return processNext(context); - } - - try { - log.debug("fetching from BrAPI service, trials belonging to required units"); - //BrAPITrialReadWorkflowInitialization brAPITrialReadWorkflowInitialization = new BrAPITrialReadWorkflowInitialization(context); - //brAPITrialReadWorkflowInitialization.execute(); - throw new ApiException(); - } catch (ApiException e) { - this.compensate(context); - } - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/FieldValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/FieldValidation.java deleted file mode 100644 index e02f021ce..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/FieldValidation.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.misc.brapi; - -import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; - -import javax.inject.Inject; - -@Slf4j -public class FieldValidation extends ExpUnitMiddleware { - ObservationUnitService observationUnitService; - - @Inject - public FieldValidation(ObservationUnitService observationUnitService) { - this.observationUnitService = observationUnitService; - } - - @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { - - try { - - } catch (Exception e) { - this.compensate(context); - } - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/ImportPreviewStatistics.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/ImportPreviewStatistics.java deleted file mode 100644 index 5848db383..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/ImportPreviewStatistics.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.misc.brapi; - -import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.misc.brapi.FieldValidation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.misc.brapi.NewPendingBrAPIObjects; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; - -import javax.inject.Inject; -import javax.inject.Provider; - -@Slf4j -public class ImportPreviewStatistics extends ExpUnitMiddleware { - ExpUnitMiddleware middleware; - private Provider newPendingBrAPIObjectsProvider; - private Provider fieldValidationProvider; - - @Inject - public ImportPreviewStatistics(Provider newPendingBrAPIObjectsProvider, - Provider fieldValidationProvider) { - - this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( - newPendingBrAPIObjectsProvider.get(), // Construct Pending import objects for new BrAPI data - fieldValidationProvider.get()); // Validate fields - } - - @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { - log.debug("generating import preview statistics"); - return this.middleware.process(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/NewPendingBrAPIObjects.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/NewPendingBrAPIObjects.java deleted file mode 100644 index 275b74e11..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/misc/brapi/NewPendingBrAPIObjects.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.misc.brapi; - -import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.ImportTableProcess; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; - -import javax.inject.Inject; - -@Slf4j -public class NewPendingBrAPIObjects extends ExpUnitMiddleware { - ExpUnitMiddleware middleware; - @Inject - public NewPendingBrAPIObjects(ImportTableProcess importTableProcess) { - this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(importTableProcess); // Construct new pending observation - } - - @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { - log.debug("constructing new pending BrAPI objects"); - - - return processNext(context); - } -} From d17a0a5b72440106c07d3569e3843fd006110f25 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:05:04 -0400 Subject: [PATCH 150/203] repackage observation validator --- .../appendoverwrite/{action => factory}/BrAPIAction.java | 3 ++- .../{action/create => factory}/BrAPICreationFactory.java | 2 +- .../{action/read => factory}/BrAPIReadFactory.java | 2 +- .../appendoverwrite/{action => factory}/BrAPIState.java | 2 +- .../{action/update => factory}/BrAPIUpdateFactory.java | 2 +- .../{action/create => factory}/WorkflowCreation.java | 7 +++---- .../read => factory}/WorkflowReadInitialization.java | 4 +--- .../{action/update => factory}/WorkflowUpdate.java | 7 +++---- .../middleware/commit/BrAPIDatasetCommit.java | 8 ++++---- .../middleware/commit/BrAPIObservationCommit.java | 8 ++++---- .../middleware/commit/BrAPIObservationUnitCommit.java | 4 ++-- .../middleware/commit/BrAPIStudyCommit.java | 4 ++-- .../middleware/commit/BrAPITrialCommit.java | 8 ++++---- .../appendoverwrite/middleware/commit/LocationCommit.java | 4 ++-- .../middleware/initialize/WorkflowInitialization.java | 4 ++-- .../middleware/process/ImportTableProcess.java | 3 +-- .../appendoverwrite/middleware/process/InitialData.java | 3 +-- .../middleware/process/OverwrittenData.java | 2 +- .../middleware/process/ProcessedDataFactory.java | 2 +- .../middleware/validator/{ => field}/DateValidator.java | 3 ++- .../middleware/validator/{ => field}/FieldValidator.java | 3 ++- .../validator/{ => field}/NominalValidator.java | 3 ++- .../validator/{ => field}/NumericalValidator.java | 3 ++- .../validator/{ => field}/ObservationValidator.java | 2 +- .../validator/{ => field}/OrdinalValidator.java | 3 ++- 25 files changed, 48 insertions(+), 48 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{action => factory}/BrAPIAction.java (92%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{action/create => factory}/BrAPICreationFactory.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{action/read => factory}/BrAPIReadFactory.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{action => factory}/BrAPIState.java (72%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{action/update => factory}/BrAPIUpdateFactory.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{action/create => factory}/WorkflowCreation.java (93%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{action/read => factory}/WorkflowReadInitialization.java (90%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{action/update => factory}/WorkflowUpdate.java (91%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/{ => field}/DateValidator.java (93%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/{ => field}/FieldValidator.java (82%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/{ => field}/NominalValidator.java (92%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/{ => field}/NumericalValidator.java (93%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/{ => field}/ObservationValidator.java (89%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/{ => field}/OrdinalValidator.java (92%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIAction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIAction.java similarity index 92% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIAction.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIAction.java index 2c00f8c17..b99505a8d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIAction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIAction.java @@ -1,7 +1,8 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; import java.util.Optional; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreationFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPICreationFactory.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreationFactory.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPICreationFactory.java index 9099607f8..c4f03b29b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/BrAPICreationFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPICreationFactory.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIReadFactory.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIReadFactory.java index f12816708..e40436e33 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/BrAPIReadFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIReadFactory.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIState.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java similarity index 72% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIState.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java index fa9355e6d..e21a0e7c8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/BrAPIState.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; public interface BrAPIState { } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdateFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIUpdateFactory.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdateFactory.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIUpdateFactory.java index 53baa720c..b00404c2d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/BrAPIUpdateFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIUpdateFactory.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/WorkflowCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowCreation.java similarity index 93% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/WorkflowCreation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowCreation.java index 1a7e8e810..8791f2654 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/create/WorkflowCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowCreation.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.server.exceptions.InternalServerException; @@ -6,10 +6,9 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIAction; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/WorkflowReadInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowReadInitialization.java similarity index 90% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/WorkflowReadInitialization.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowReadInitialization.java index 5faaaadc3..0dcd73e3c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/read/WorkflowReadInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowReadInitialization.java @@ -1,11 +1,9 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; import io.micronaut.context.annotation.Prototype; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; import org.breedinginsight.utilities.Utilities; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/WorkflowUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowUpdate.java similarity index 91% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/WorkflowUpdate.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowUpdate.java index b15e2976d..3cac7d608 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/action/update/WorkflowUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowUpdate.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.server.exceptions.InternalServerException; @@ -6,10 +6,9 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIAction; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIAction; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; import java.util.List; import java.util.Optional; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index 75c0b4671..02f2a5956 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -4,10 +4,10 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIUpdateFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.WorkflowUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIUpdateFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java index 2220f5a6c..2bede0002 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -4,10 +4,10 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIUpdateFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.WorkflowUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIUpdateFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index 64f700f22..5b0abadae 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -4,8 +4,8 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java index d4a08738e..80c2d555f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -4,8 +4,8 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index 40fbb7a47..39e4c0588 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -4,10 +4,10 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.BrAPIUpdateFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.update.WorkflowUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIUpdateFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index 8857f5445..922d273d2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -3,8 +3,8 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.create.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index 012551c63..3b3ec2c63 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -8,8 +8,8 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.BrAPIReadFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.action.read.WorkflowReadInitialization; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIReadFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowReadInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 068b130f4..7c86f76a8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -22,10 +22,9 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.FileMappingUtil; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java index 83e48dfdb..978a075c4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java @@ -1,7 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; import com.google.gson.Gson; -import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.pheno.BrAPIObservation; @@ -10,7 +9,7 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; import org.breedinginsight.model.Program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java index 16706c0d2..a00acfcfc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java @@ -11,7 +11,7 @@ import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.FieldValidator; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; import org.breedinginsight.utilities.Utilities; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java index ff6c17a14..ad3db2cf4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java @@ -7,7 +7,7 @@ import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; import org.breedinginsight.model.Program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/DateValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/DateValidator.java similarity index 93% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/DateValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/DateValidator.java index 241696289..392dee7dc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/DateValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/DateValidator.java @@ -1,8 +1,9 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.ObservationValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.model.Trait; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/FieldValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/FieldValidator.java similarity index 82% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/FieldValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/FieldValidator.java index de4c7dee4..61c7ebdb5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/FieldValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/FieldValidator.java @@ -1,7 +1,8 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field; import io.micronaut.context.annotation.Primary; import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.ObservationValidator; import org.breedinginsight.model.Trait; import javax.inject.Singleton; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/NominalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/NominalValidator.java similarity index 92% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/NominalValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/NominalValidator.java index 2416d6d30..43289f6ae 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/NominalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/NominalValidator.java @@ -1,8 +1,9 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.ObservationValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.model.Trait; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/NumericalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/NumericalValidator.java similarity index 93% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/NumericalValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/NumericalValidator.java index 13a95062f..59a6d4f34 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/NumericalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/NumericalValidator.java @@ -1,8 +1,9 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.ObservationValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.model.Trait; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/ObservationValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/ObservationValidator.java similarity index 89% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/ObservationValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/ObservationValidator.java index 0ff664d03..2b46057ef 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/ObservationValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/ObservationValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field; import io.micronaut.core.annotation.NonNull; import io.micronaut.core.order.Ordered; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/OrdinalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/OrdinalValidator.java similarity index 92% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/OrdinalValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/OrdinalValidator.java index 90c2967c2..4b3acc9fe 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/OrdinalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/OrdinalValidator.java @@ -1,8 +1,9 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.ObservationValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.model.Trait; From 2319cbf4fb8db29ebb5ed32452e71ee52f097696 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:49:45 -0400 Subject: [PATCH 151/203] repackage factories --- .../appendoverwrite/middleware/ExpUnitMiddleware.java | 2 +- .../middleware/commit/BrAPIDatasetCommit.java | 8 ++++---- .../middleware/commit/BrAPIObservationCommit.java | 8 ++++---- .../middleware/commit/BrAPIObservationUnitCommit.java | 4 ++-- .../middleware/commit/BrAPIStudyCommit.java | 4 ++-- .../middleware/commit/BrAPITrialCommit.java | 8 ++++---- .../middleware/commit/LocationCommit.java | 4 ++-- .../middleware/initialize/WorkflowInitialization.java | 4 ++-- .../middleware/process/ImportTableProcess.java | 2 +- .../appendoverwrite/middleware/process/InitialData.java | 2 +- .../middleware/process/OverwrittenData.java | 2 +- .../middleware/process/ProcessedDataFactory.java | 2 +- .../{appendoverwrite => }/factory/BrAPIState.java | 2 +- .../factory => factory/action}/BrAPIAction.java | 6 +++--- .../factory => factory/action}/BrAPICreationFactory.java | 4 ++-- .../factory => factory/action}/BrAPIReadFactory.java | 4 ++-- .../factory => factory/action}/BrAPIUpdateFactory.java | 4 ++-- .../factory => factory/action}/WorkflowCreation.java | 7 +++---- .../action}/WorkflowReadInitialization.java | 5 +++-- .../factory => factory/action}/WorkflowUpdate.java | 7 +++---- .../entity/ExperimentImportEntity.java | 2 +- .../entity/PendingDataset.java | 3 +-- .../entity/PendingEntityFactory.java | 9 +++++---- .../entity/PendingGermplasm.java | 3 +-- .../entity/PendingLocation.java | 3 +-- .../entity/PendingObservation.java | 4 +--- .../entity/PendingObservationUnit.java | 3 +-- .../entity/PendingStudy.java | 4 +--- .../entity/PendingTrial.java | 4 +--- .../experiment/{ => model}/middleware/Middleware.java | 2 +- .../middleware => }/validator/field/DateValidator.java | 3 +-- .../middleware => }/validator/field/FieldValidator.java | 3 +-- .../validator/field/NominalValidator.java | 3 +-- .../validator/field/NumericalValidator.java | 3 +-- .../validator/field/ObservationValidator.java | 2 +- .../validator/field/OrdinalValidator.java | 3 +-- 36 files changed, 64 insertions(+), 79 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite => }/factory/BrAPIState.java (72%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite/factory => factory/action}/BrAPIAction.java (92%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite/factory => factory/action}/BrAPICreationFactory.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite/factory => factory/action}/BrAPIReadFactory.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite/factory => factory/action}/BrAPIUpdateFactory.java (96%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite/factory => factory/action}/WorkflowCreation.java (92%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite/factory => factory/action}/WorkflowReadInitialization.java (93%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite/factory => factory/action}/WorkflowUpdate.java (91%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite => factory}/entity/ExperimentImportEntity.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite => factory}/entity/PendingDataset.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite => factory}/entity/PendingEntityFactory.java (94%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite => factory}/entity/PendingGermplasm.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite => factory}/entity/PendingLocation.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite => factory}/entity/PendingObservation.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite => factory}/entity/PendingObservationUnit.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite => factory}/entity/PendingStudy.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite => factory}/entity/PendingTrial.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => model}/middleware/Middleware.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite/middleware => }/validator/field/DateValidator.java (93%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite/middleware => }/validator/field/FieldValidator.java (82%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite/middleware => }/validator/field/NominalValidator.java (92%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite/middleware => }/validator/field/NumericalValidator.java (93%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite/middleware => }/validator/field/ObservationValidator.java (89%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{appendoverwrite/middleware => }/validator/field/OrdinalValidator.java (92%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java index 4763f10dd..6e0cacc80 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java @@ -1,6 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.Middleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.middleware.Middleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index 02f2a5956..6d40977e1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -4,10 +4,10 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIUpdateFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPIUpdateFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java index 2bede0002..f7217f3a7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -4,10 +4,10 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIUpdateFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPIUpdateFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index 5b0abadae..d72f03e7a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -4,8 +4,8 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java index 80c2d555f..c621e988c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -4,8 +4,8 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index 39e4c0588..4008f5b2a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -4,10 +4,10 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIUpdateFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPIUpdateFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index 922d273d2..e5726cf50 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -3,8 +3,8 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index 3b3ec2c63..7d786dac6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -8,8 +8,8 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIReadFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.WorkflowReadInitialization; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPIReadFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowReadInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 7c86f76a8..47536de35 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -24,7 +24,7 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java index 978a075c4..8a953e2ff 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java @@ -9,7 +9,7 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; import org.breedinginsight.model.Program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java index a00acfcfc..8d924e091 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java @@ -11,7 +11,7 @@ import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; import org.breedinginsight.utilities.Utilities; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java index ad3db2cf4..9b5902429 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java @@ -7,7 +7,7 @@ import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; import org.breedinginsight.model.Program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/BrAPIState.java similarity index 72% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/BrAPIState.java index e21a0e7c8..9cd21415d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/BrAPIState.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; +package org.breedinginsight.brapps.importer.services.processors.experiment.factory; public interface BrAPIState { } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIAction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIAction.java similarity index 92% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIAction.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIAction.java index b99505a8d..0c63b7618 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIAction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIAction.java @@ -1,8 +1,8 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; +package org.breedinginsight.brapps.importer.services.processors.experiment.factory.action; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity.ExperimentImportEntity; import java.util.Optional; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPICreationFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPICreationFactory.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPICreationFactory.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPICreationFactory.java index c4f03b29b..db5c6f4bf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPICreationFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPICreationFactory.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; +package org.breedinginsight.brapps.importer.services.processors.experiment.factory.action; import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; @@ -8,7 +8,7 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.model.ProgramLocation; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIReadFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIReadFactory.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIReadFactory.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIReadFactory.java index e40436e33..07ab687d2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIReadFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIReadFactory.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; +package org.breedinginsight.brapps.importer.services.processors.experiment.factory.action; import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; @@ -8,7 +8,7 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.model.ProgramLocation; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIUpdateFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIUpdateFactory.java similarity index 96% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIUpdateFactory.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIUpdateFactory.java index b00404c2d..02c65a69f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIUpdateFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIUpdateFactory.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; +package org.breedinginsight.brapps.importer.services.processors.experiment.factory.action; import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; @@ -6,7 +6,7 @@ import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.BrAPIObservation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.PendingEntityFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowCreation.java similarity index 92% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowCreation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowCreation.java index 8791f2654..e3ab70dc5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowCreation.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; +package org.breedinginsight.brapps.importer.services.processors.experiment.factory.action; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.server.exceptions.InternalServerException; @@ -6,9 +6,8 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIAction; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity.ExperimentImportEntity; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowReadInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowReadInitialization.java similarity index 93% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowReadInitialization.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowReadInitialization.java index 0dcd73e3c..2f72aa7fc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowReadInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowReadInitialization.java @@ -1,10 +1,11 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; +package org.breedinginsight.brapps.importer.services.processors.experiment.factory.action; import io.micronaut.context.annotation.Prototype; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity.ExperimentImportEntity; import org.breedinginsight.utilities.Utilities; import java.util.List; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowUpdate.java similarity index 91% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowUpdate.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowUpdate.java index 3cac7d608..85535d40f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/WorkflowUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowUpdate.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; +package org.breedinginsight.brapps.importer.services.processors.experiment.factory.action; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.server.exceptions.InternalServerException; @@ -6,9 +6,8 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIAction; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity.ExperimentImportEntity; import java.util.List; import java.util.Optional; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/ExperimentImportEntity.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/ExperimentImportEntity.java index 6edf2afc6..63b4d4c87 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/ExperimentImportEntity.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/ExperimentImportEntity.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingDataset.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingDataset.java index 83f95df92..f53ba7d99 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingDataset.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; @@ -18,7 +18,6 @@ import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import javax.inject.Inject; import java.util.ArrayList; import java.util.List; import java.util.Map; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingEntityFactory.java similarity index 94% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingEntityFactory.java index b73b12415..d430b681e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingEntityFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingEntityFactory.java @@ -1,10 +1,11 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Prototype; import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity.*; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.*; import org.breedinginsight.services.OntologyService; @@ -66,9 +67,9 @@ public static PendingTrial pendingTrial(ExpUnitMiddlewareContext context, } public static PendingObservationUnit pendingObservationUnit(ExpUnitMiddlewareContext context, - BrAPIObservationUnitDAO observationUnitDAO, - ObservationUnitService observationUnitService, - ExperimentUtilities experimentUtilities) { + BrAPIObservationUnitDAO observationUnitDAO, + ObservationUnitService observationUnitService, + ExperimentUtilities experimentUtilities) { return new PendingObservationUnit(context, observationUnitDAO, observationUnitService, experimentUtilities); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingGermplasm.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingGermplasm.java index 3b01581bd..66a13e13f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingGermplasm.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; @@ -15,7 +15,6 @@ import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import javax.inject.Inject; import java.util.*; import java.util.stream.Collectors; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingLocation.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingLocation.java index e9e12ffc8..42d1645c1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingLocation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingLocation.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; @@ -18,7 +18,6 @@ import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import javax.inject.Inject; import java.util.*; import java.util.stream.Collectors; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingObservation.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingObservation.java index 686e507e7..52cbedd90 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingObservation.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; import io.micronaut.context.annotation.Prototype; import org.apache.commons.collections4.map.CaseInsensitiveMap; @@ -10,12 +10,10 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.model.Trait; import org.breedinginsight.services.OntologyService; import org.breedinginsight.services.exceptions.DoesNotExistException; -import javax.inject.Inject; import java.util.ArrayList; import java.util.List; import java.util.Optional; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingObservationUnit.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingObservationUnit.java index ea6efea3e..714374058 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingObservationUnit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingObservationUnit.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; import io.micronaut.context.annotation.Prototype; import org.apache.commons.lang3.StringUtils; @@ -18,7 +18,6 @@ import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; -import javax.inject.Inject; import java.util.*; import java.util.stream.Collectors; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingStudy.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingStudy.java index 2beb33a7b..27003e534 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingStudy.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingStudy.java @@ -1,9 +1,8 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -17,7 +16,6 @@ import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; -import javax.inject.Inject; import java.util.*; import java.util.stream.Collectors; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingTrial.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingTrial.java index db777460a..7bf3ce3a4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/entity/PendingTrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingTrial.java @@ -1,9 +1,8 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -14,7 +13,6 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; import org.breedinginsight.utilities.Utilities; -import javax.inject.Inject; import java.util.*; import java.util.stream.Collectors; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/middleware/Middleware.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/middleware/Middleware.java index b4757cb17..aa60d27c6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/middleware/Middleware.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.middleware; +package org.breedinginsight.brapps.importer.services.processors.experiment.model.middleware; public abstract class Middleware { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/DateValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java similarity index 93% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/DateValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java index 392dee7dc..21b0e8448 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/DateValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java @@ -1,9 +1,8 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.model.v1.response.ValidationError; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.ObservationValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.model.Trait; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/FieldValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/FieldValidator.java similarity index 82% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/FieldValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/FieldValidator.java index 61c7ebdb5..01253ef97 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/FieldValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/FieldValidator.java @@ -1,8 +1,7 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; import io.micronaut.context.annotation.Primary; import org.breedinginsight.api.model.v1.response.ValidationError; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.ObservationValidator; import org.breedinginsight.model.Trait; import javax.inject.Singleton; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/NominalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NominalValidator.java similarity index 92% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/NominalValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NominalValidator.java index 43289f6ae..bd2070002 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/NominalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NominalValidator.java @@ -1,9 +1,8 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.model.v1.response.ValidationError; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.ObservationValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.model.Trait; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/NumericalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java similarity index 93% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/NumericalValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java index 59a6d4f34..5a4eba21d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/NumericalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java @@ -1,9 +1,8 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.model.v1.response.ValidationError; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.ObservationValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.model.Trait; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/ObservationValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/ObservationValidator.java similarity index 89% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/ObservationValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/ObservationValidator.java index 2b46057ef..b28528316 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/ObservationValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/ObservationValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; import io.micronaut.core.annotation.NonNull; import io.micronaut.core.order.Ordered; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/OrdinalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/OrdinalValidator.java similarity index 92% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/OrdinalValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/OrdinalValidator.java index 4b3acc9fe..4a3077077 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validator/field/OrdinalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/OrdinalValidator.java @@ -1,9 +1,8 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.model.v1.response.ValidationError; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validator.field.ObservationValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.model.Trait; From 3e722341b0d1b010dc72e31e29ecf1cf1cdcfe0b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 21 Jun 2024 19:19:27 -0400 Subject: [PATCH 152/203] delete unused prcessor --- .../workflow => model}/Workflow.java | 2 +- .../experiment/ExperimentProcessor.java | 42 ---------- .../experiment/service/DatasetService.java | 79 ++----------------- .../AppendOverwritePhenotypesWorkflow.java | 40 ---------- .../workflow/ExperimentWorkflowFactory.java | 70 ---------------- 5 files changed, 6 insertions(+), 227 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/{services/processors/experiment/workflow => model}/Workflow.java (77%) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentProcessor.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/AppendOverwritePhenotypesWorkflow.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/Workflow.java similarity index 77% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/Workflow.java rename to src/main/java/org/breedinginsight/brapps/importer/model/Workflow.java index 6c0384a0f..807f8387f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/Workflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/Workflow.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.workflow; +package org.breedinginsight.brapps.importer.model; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentProcessor.java deleted file mode 100644 index 8207086fe..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentProcessor.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment; - -import io.micronaut.context.annotation.Prototype; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; -import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.ExperimentWorkflowFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.Workflow; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.ValidatorException; - -import javax.inject.Inject; -import java.util.HashMap; -import java.util.Map; - -@Slf4j -@Prototype -public class ExperimentProcessor { - - private final ExperimentWorkflowFactory experimentWorkflowFactory; - - @Inject - public ExperimentProcessor(ExperimentWorkflowFactory experimentWorkflowFactory) { - this.experimentWorkflowFactory = experimentWorkflowFactory; - } - - public Map process(ImportContext context) - throws ApiException, ValidatorException, MissingRequiredInfoException, UnprocessableEntityException { - - // determine which workflow to use based on the import context - Workflow workflow = experimentWorkflowFactory.getWorkflow(context); - log.info("Importing experiment data using workflow: " + workflow.getName()); - - ProcessedData output = workflow.process(context); - - - return new HashMap<>(); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index b95a6d399..a4cc47aad 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -7,27 +7,22 @@ import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIListTypes; import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.brapps.importer.services.processors.ProcessorData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; -import java.util.*; -import java.util.stream.Collectors; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; @Singleton public class DatasetService { @@ -101,39 +96,7 @@ public PendingImportObject constructPIOFromDataset(BrAPIListDe return new PendingImportObject(ImportObjectState.EXISTING, dataset, UUID.fromString(xref.getReferenceId())); } - // TODO: used by expunit worflow -// public Map> initializeObsVarDatasetForExistingObservationUnits( -// Map> trialByName, -// Program program) { -// Map> obsVarDatasetByName = new HashMap<>(); -// -// if (trialByName.size() > 0 && -// trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { -// String datasetId = trialByName.values().iterator().next().getBrAPIObject() -// .getAdditionalInfo() -// .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) -// .getAsString(); -// -// try { -// List existingDatasets = brAPIListDAO -// .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, -// program.getId(), -// String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), -// UUID.fromString(datasetId)); -// if (existingDatasets == null || existingDatasets.isEmpty()) { -// throw new InternalServerException("existing dataset summary not returned from brapi server"); -// } -// BrAPIListDetails dataSetDetails = brAPIListDAO -// .getListById(existingDatasets.get(0).getListDbId(), program.getId()) -// .getResult(); -// processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); -// } catch (ApiException e) { -// log.error(Utilities.generateApiExceptionLogMessage(e), e); -// throw new InternalServerException(e.toString(), e); -// } -// } -// return obsVarDatasetByName; -// } + // TODO: used by create workflow // public Map> initializeObsVarDatasetByName(Program program, List experimentImportRows) { @@ -199,38 +162,6 @@ public void addObsVarsToDatasetDetails(PendingImportObject pio }); } - // TODO: used by expunit workflow -// public void fetchOrCreateDatasetPIO(ImportContext importContext, -// PendingData pendingData, -// ExpUnitContext expUnitContext, -// List referencedTraits) throws UnprocessableEntityException { -// PendingImportObject pio; -// PendingImportObject trialPIO = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); -// String name = String.format("Observation Dataset [%s-%s]", -// program.getKey(), -// trialPIO.getBrAPIObject() -// .getAdditionalInfo() -// .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) -// .getAsString()); -// if (obsVarDatasetByName.containsKey(name)) { -// pio = obsVarDatasetByName.get(name); -// } else { -// UUID id = UUID.randomUUID(); -// BrAPIListDetails newDataset = importRow.constructDatasetDetails( -// name, -// id, -// BRAPI_REFERENCE_SOURCE, -// program, -// trialPIO.getId().toString()); -// pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); -// trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); -// if (ImportObjectState.EXISTING == trialPIO.getState()) { -// trialPIO.setState(ImportObjectState.MUTATED); -// } -// obsVarDatasetByName.put(name, pio); -// } -// addObsVarsToDatasetDetails(pio, referencedTraits, program); -// } // TODO: used by create workflow // public void fetchOrCreateDatasetPIO(ImportContext importContext, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/AppendOverwritePhenotypesWorkflow.java deleted file mode 100644 index a682edaca..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/AppendOverwritePhenotypesWorkflow.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.workflow; - -import io.micronaut.context.annotation.Prototype; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.GetExistingBrAPIData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.Transaction; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitIDValidation; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; - -import javax.inject.Inject; - - -@Prototype -public class AppendOverwritePhenotypesWorkflow implements Workflow { - - ExpUnitMiddleware middleware; - @Inject - public AppendOverwritePhenotypesWorkflow(Transaction transaction, - ExpUnitIDValidation expUnitIDValidation, - GetExistingBrAPIData getExistingBrAPIData) { - - this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(transaction, expUnitIDValidation, getExistingBrAPIData); - } - @Override - public ProcessedData process(ImportContext context) { - ExpUnitMiddlewareContext workflowContext = ExpUnitMiddlewareContext.builder().importContext(context).build(); - this.middleware.process(workflowContext); - - // TODO: implement - return new ProcessedData(); - } - - @Override - public String getName() { - return "AppendOverwritePhenotypesWorkflow"; - } - -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java deleted file mode 100644 index 0d4c7a698..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.workflow; - -import org.apache.commons.lang3.StringUtils; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.CreateNewExperimentWorkflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; - -import javax.inject.Inject; -import javax.inject.Provider; -import javax.inject.Singleton; -import java.util.List; - -@Singleton -public class ExperimentWorkflowFactory { - - private final Provider createNewExperimentWorkflowProvider; - private final Provider appendOverwritePhenotypesWorkflowProvider; - - @Inject - public ExperimentWorkflowFactory(Provider createNewExperimentWorkflowProvider, - Provider appendOverwritePhenotypesWorkflowProvider) { - this.createNewExperimentWorkflowProvider = createNewExperimentWorkflowProvider; - this.appendOverwritePhenotypesWorkflowProvider = appendOverwritePhenotypesWorkflowProvider; - } - - /** - * Retrieves the appropriate workflow based on the provided import context. Validation will be done - * in selected workflow, not here. For example will not check if file has ObsUnitIDs that all rows have one. - * We are just checking the basic condition for what type of workflow to return. - * - * @param context import context containing import rows - * @return the workflow to be used for processing the import rows - */ - public Workflow getWorkflow(ImportContext context) { - - List importRows = context.getImportRows(); - - boolean hasExpUnitObsUnitIDs = importRows.stream() - .anyMatch(row -> { - ExperimentObservation expRow = (ExperimentObservation) row; - return StringUtils.isNotBlank(expRow.getObsUnitID()); - }); - - if (hasExpUnitObsUnitIDs) { - long distinctCount = importRows.stream() - .map(row -> { - ExperimentObservation expRow = (ExperimentObservation) row; - return expRow.getObsUnitID(); - }) - .distinct() - .count(); - - if (distinctCount != importRows.size()) { - // If have ExpUnit ObsUnitIDs and there are duplicates -> Append / Update SubObsUnit Phenotypes - // TODO: different workflow for subobs units? - return appendOverwritePhenotypesWorkflowProvider.get(); - } else { - // If have ExpUnit ObsUnitIDs and all are unique -> Append / Update ExpUnit Phenotypes - return appendOverwritePhenotypesWorkflowProvider.get(); - } - - } else { - // No ObsUnitIDs so creating experiment or appending env - return (Workflow) createNewExperimentWorkflowProvider.get(); - // TODO: different workflow for appending envs? Would have a dependency on DAO to check for existing trial name - } - } - -} From b235495900245d7cb070b4cb46ced706ecdee840 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 21 Jun 2024 19:25:59 -0400 Subject: [PATCH 153/203] delete unused pipeline package --- .../experiment/pipeline/Pipeline.java | 18 ------------------ .../experiment/pipeline/ProcessingStep.java | 5 ----- 2 files changed, 23 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/Pipeline.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/ProcessingStep.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/Pipeline.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/Pipeline.java deleted file mode 100644 index 09b657a89..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/Pipeline.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.pipeline; - -public class Pipeline { - - private final ProcessingStep currentStep; - - public Pipeline(ProcessingStep currentStep) { - this.currentStep = currentStep; - } - - public Pipeline addProcessingStep(ProcessingStep newStep) { - return new Pipeline<>(input -> newStep.process(currentStep.process(input))); - } - - public O execute(I input) { - return currentStep.process(input); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/ProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/ProcessingStep.java deleted file mode 100644 index 2407e646e..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/ProcessingStep.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.pipeline; - -public interface ProcessingStep { - O process(I input); -} From 352061f95a28768fa7f4a215bd7e14de7dcd34a9 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 21 Jun 2024 19:32:04 -0400 Subject: [PATCH 154/203] delete unused expunitcontext service --- .../ExpUnitContextService.java | 107 ------------------ .../factory/BrAPIState.java | 2 +- .../factory/action/BrAPIAction.java | 6 +- .../factory/action/BrAPICreationFactory.java | 4 +- .../factory/action/BrAPIReadFactory.java | 4 +- .../factory/action/BrAPIUpdateFactory.java | 4 +- .../factory/action/WorkflowCreation.java | 6 +- .../action/WorkflowReadInitialization.java | 6 +- .../factory/action/WorkflowUpdate.java | 6 +- .../entity/ExperimentImportEntity.java | 2 +- .../factory/entity/PendingDataset.java | 2 +- .../factory/entity/PendingEntityFactory.java | 2 +- .../factory/entity/PendingGermplasm.java | 2 +- .../factory/entity/PendingLocation.java | 2 +- .../factory/entity/PendingObservation.java | 2 +- .../entity/PendingObservationUnit.java | 2 +- .../factory/entity/PendingStudy.java | 2 +- .../factory/entity/PendingTrial.java | 2 +- .../middleware/commit/BrAPIDatasetCommit.java | 8 +- .../commit/BrAPIObservationCommit.java | 8 +- .../commit/BrAPIObservationUnitCommit.java | 4 +- .../middleware/commit/BrAPIStudyCommit.java | 4 +- .../middleware/commit/BrAPITrialCommit.java | 8 +- .../middleware/commit/LocationCommit.java | 4 +- .../initialize/WorkflowInitialization.java | 4 +- 25 files changed, 48 insertions(+), 155 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/factory/BrAPIState.java (72%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/factory/action/BrAPIAction.java (92%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/factory/action/BrAPICreationFactory.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/factory/action/BrAPIReadFactory.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/factory/action/BrAPIUpdateFactory.java (95%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/factory/action/WorkflowCreation.java (95%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/factory/action/WorkflowReadInitialization.java (94%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/factory/action/WorkflowUpdate.java (94%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/factory/entity/ExperimentImportEntity.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/factory/entity/PendingDataset.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/factory/entity/PendingEntityFactory.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/factory/entity/PendingGermplasm.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/factory/entity/PendingLocation.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/factory/entity/PendingObservation.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/factory/entity/PendingObservationUnit.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/factory/entity/PendingStudy.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/factory/entity/PendingTrial.java (99%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java deleted file mode 100644 index fed70bb51..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java +++ /dev/null @@ -1,107 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite; - -import io.micronaut.context.annotation.Property; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.model.Program; -import org.breedinginsight.utilities.Utilities; - -import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; - -@Slf4j -public class ExpUnitContextService { - private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; - @Property(name = "brapi.server.reference-source") - private String BRAPI_REFERENCE_SOURCE; - - @Inject - public ExpUnitContextService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { - this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; - } - public List getReferenceUnits(Set expUnitIds, - Program program) throws ApiException { - // Retrieve reference Observation Units based on IDs - return brAPIObservationUnitDAO.getObservationUnitsById(new ArrayList(expUnitIds), program); - } - - public PendingImportObject constructPIOFromExistingUnit(BrAPIObservationUnit unit) { - final PendingImportObject[] pio = new PendingImportObject[]{null}; - - // Construct the DeltaBreed observation unit source for external references - String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - - // Get external reference for the Observation Unit - Optional unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource); - unitXref.ifPresentOrElse( - xref -> { - pio[0] = new PendingImportObject(ImportObjectState.EXISTING, unit, UUID.fromString(xref.getReferenceId())); - }, - () -> { - - // but throw an error if no unit ID - throw new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID"); - } - ); - return pio[0]; - } - - public Map> mapPendingUnitById(List> pios) { - Map> pendingUnitById = new HashMap<>(); - - // Construct the DeltaBreed observation unit source for external references - String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - - for (PendingImportObject pio : pios) { - - // Get external reference for the Observation Unit - Optional xref = Utilities.getExternalReference(pio.getBrAPIObject().getExternalReferences(), deltaBreedOUSource); - pendingUnitById.put(xref.get().getReferenceId(),pio); - } - - return pendingUnitById; - } - - public Map> mapPendingUnitByNameNoScope(List> pios, - Program program) { - Map> pendingUnitByNameNoScope = new HashMap<>(); - - for (PendingImportObject pio : pios) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( - pio.getBrAPIObject().getStudyName(), - program.getKey() - ); - String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( - pio.getBrAPIObject().getObservationUnitName(), - program.getKey() - ); - pendingUnitByNameNoScope.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName), pio); - } - - return pendingUnitByNameNoScope; - } - - - public List collectMissingOUIds(Set referenceIds, List existingUnits) { - List missingIds = new ArrayList<>(referenceIds); - - // Construct the DeltaBreed observation unit source for external references - String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - - Set fetchedIds = existingUnits.stream() - .filter(unit ->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).isPresent()) - .map(unit->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).get().getReferenceId()) - .collect(Collectors.toSet()); - missingIds.removeAll(fetchedIds); - - return missingIds; - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/BrAPIState.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java similarity index 72% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/BrAPIState.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java index 9cd21415d..e21a0e7c8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/BrAPIState.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.factory; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; public interface BrAPIState { } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIAction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java similarity index 92% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIAction.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java index 0c63b7618..0f2af6bd2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIAction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java @@ -1,8 +1,8 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.factory.action; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.BrAPIState; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity; import java.util.Optional; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPICreationFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPICreationFactory.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java index db5c6f4bf..fdfee9a5e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPICreationFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.factory.action; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; @@ -8,7 +8,7 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity.PendingEntityFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.model.ProgramLocation; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIReadFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIReadFactory.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java index 07ab687d2..5470c557f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIReadFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.factory.action; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; @@ -8,7 +8,7 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity.PendingEntityFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.model.ProgramLocation; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIUpdateFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java similarity index 95% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIUpdateFactory.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java index 02c65a69f..f8796f708 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/BrAPIUpdateFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.factory.action; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; @@ -6,7 +6,7 @@ import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.BrAPIObservation; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity.PendingEntityFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java similarity index 95% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowCreation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java index e3ab70dc5..571f0b8d6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.factory.action; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.server.exceptions.InternalServerException; @@ -6,8 +6,8 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.BrAPIState; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowReadInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java similarity index 94% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowReadInitialization.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java index 2f72aa7fc..caeedd3cd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowReadInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java @@ -1,11 +1,11 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.factory.action; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; import io.micronaut.context.annotation.Prototype; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.BrAPIState; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity; import org.breedinginsight.utilities.Utilities; import java.util.List; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java similarity index 94% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowUpdate.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java index 85535d40f..95de8f367 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/action/WorkflowUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.factory.action; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.server.exceptions.InternalServerException; @@ -6,8 +6,8 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.BrAPIState; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity; import java.util.List; import java.util.Optional; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/ExperimentImportEntity.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/ExperimentImportEntity.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java index 63b4d4c87..1f37a3425 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/ExperimentImportEntity.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingDataset.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java index f53ba7d99..b628ecf4d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingEntityFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingEntityFactory.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java index d430b681e..4a4b4964b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingEntityFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingGermplasm.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java index 66a13e13f..f0e3daff5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingLocation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingLocation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java index 42d1645c1..08ba57159 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingLocation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingObservation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java index 52cbedd90..efc675a7b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import io.micronaut.context.annotation.Prototype; import org.apache.commons.collections4.map.CaseInsensitiveMap; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingObservationUnit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingObservationUnit.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java index 714374058..d197acfc6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingObservationUnit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import io.micronaut.context.annotation.Prototype; import org.apache.commons.lang3.StringUtils; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingStudy.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingStudy.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java index 27003e534..ee85b283e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingStudy.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingTrial.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java index 7bf3ce3a4..a34e9d349 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/factory/entity/PendingTrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import io.micronaut.context.annotation.Prototype; import org.brapi.client.v2.model.exceptions.ApiException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index 6d40977e1..abd6dd3e1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -4,10 +4,10 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPIUpdateFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java index f7217f3a7..12eec0623 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -4,10 +4,10 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservation; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPIUpdateFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index d72f03e7a..f1a23dd82 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -4,8 +4,8 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java index c621e988c..136bed8b2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -4,8 +4,8 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index 4008f5b2a..d296c9037 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -4,10 +4,10 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPIUpdateFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index e5726cf50..566d3667b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -3,8 +3,8 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index 7d786dac6..11c7f6fbe 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -8,8 +8,8 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.BrAPIReadFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.action.WorkflowReadInitialization; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIReadFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowReadInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; From 72be0bdaef4a83558325765235d75c1b58c892a7 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 21 Jun 2024 20:08:32 -0400 Subject: [PATCH 155/203] move data factory --- .../AppendOverwritePhenotypesWorkflow.java | 2 +- .../process => factory/data}/InitialData.java | 4 +- .../data}/OverwrittenData.java | 4 +- .../data}/ProcessedDataFactory.java | 2 +- .../data}/UnchangedData.java | 4 +- .../data}/VisitedObservationData.java | 3 +- .../factory/entity/PendingEntityFactory.java | 1 - .../middleware/ExpUnitIDValidation.java | 1 + .../middleware/GetExistingBrAPIData.java | 1 + .../middleware/Transaction.java | 1 + .../middleware/commit/BrAPICommit.java | 2 +- .../middleware/commit/BrAPIDatasetCommit.java | 4 +- .../commit/BrAPIObservationCommit.java | 4 +- .../commit/BrAPIObservationUnitCommit.java | 4 +- .../middleware/commit/BrAPIStudyCommit.java | 4 +- .../middleware/commit/BrAPITrialCommit.java | 4 +- .../middleware/commit/LocationCommit.java | 4 +- .../initialize/WorkflowInitialization.java | 4 +- .../process/ImportTableProcess.java | 6 +- .../appendoverwrite/model/ExpUnitContext.java | 1 - .../ExpUnitMiddleware.java | 3 +- .../model}/Middleware.java | 2 +- .../model/MiddlewareError.java | 2 +- .../steps/GetExistingProcessingStep.java | 472 ------------------ .../experiment/create/steps/ProcessStep.java | 16 - .../service/ObservationVariableService.java | 18 - 26 files changed, 38 insertions(+), 535 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{middleware/process => factory/data}/InitialData.java (94%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{middleware/process => factory/data}/OverwrittenData.java (96%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{middleware/process => factory/data}/ProcessedDataFactory.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{middleware/process => factory/data}/UnchangedData.java (83%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{middleware/process => factory/data}/VisitedObservationData.java (77%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/{middleware => model}/ExpUnitMiddleware.java (88%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{model/middleware => appendoverwrite/model}/Middleware.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/model/MiddlewareError.java (93%) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index ece1e8672..553f70316 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -10,7 +10,7 @@ import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; import org.breedinginsight.brapps.importer.services.ImportStatusService; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.Transaction; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitIDValidation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.BrAPICommit; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java similarity index 94% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java index 8a953e2ff..5d5dc1403 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; import com.google.gson.Gson; import io.micronaut.context.annotation.Prototype; @@ -9,6 +9,8 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.VisitedObservationData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java similarity index 96% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java index 8d924e091..93ca0f421 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -11,6 +11,8 @@ import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.VisitedObservationData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java index 9b5902429..b744a3ba4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ProcessedDataFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UnchangedData.java similarity index 83% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UnchangedData.java index 8cabd3045..9f59d3251 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UnchangedData.java @@ -1,10 +1,12 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; import io.micronaut.context.annotation.Prototype; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.VisitedObservationData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/VisitedObservationData.java similarity index 77% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/VisitedObservationData.java index bdd064b93..13c1f1695 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/VisitedObservationData.java @@ -1,8 +1,9 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; import java.util.List; import java.util.Optional; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java index 4a4b4964b..b65ee97f2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java @@ -5,7 +5,6 @@ import io.micronaut.context.annotation.Prototype; import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.factory.entity.*; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.*; import org.breedinginsight.services.OntologyService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java index 5848b1dbd..6a945692c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java @@ -3,6 +3,7 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; @Slf4j diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java index 5e6c52a5b..f48706375 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java @@ -10,6 +10,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.utilities.Utilities; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java index 50006293e..7d13227da 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java @@ -2,6 +2,7 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; @Slf4j diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java index 076e18c54..7b7a964fc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java @@ -2,7 +2,7 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index abd6dd3e1..5a677a993 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -8,9 +8,9 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import javax.inject.Inject; import java.util.Optional; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java index 12eec0623..bb583dba5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -8,9 +8,9 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import javax.inject.Inject; import java.util.Optional; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index f1a23dd82..8b1b842b4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -6,9 +6,9 @@ import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import javax.inject.Inject; import java.util.Optional; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java index 136bed8b2..ccf47edfc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -6,9 +6,9 @@ import org.brapi.v2.model.core.BrAPIStudy; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import javax.inject.Inject; import java.util.Optional; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index d296c9037..d9e41f7be 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -8,9 +8,9 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import javax.inject.Inject; import java.util.Optional; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index 566d3667b..1438d0fe8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -5,9 +5,9 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import org.breedinginsight.model.ProgramLocation; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index 11c7f6fbe..e9b0230a2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -10,9 +10,9 @@ import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIReadFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowReadInitialization; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import org.breedinginsight.model.ProgramLocation; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 47536de35..c59dc0ebe 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -23,10 +23,12 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.ProcessedDataFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.VisitedObservationData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationVariableService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java index 000ad0680..41990c8e1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java @@ -11,7 +11,6 @@ import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.model.ProgramLocation; import tech.tablesaw.columns.Column; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitMiddleware.java similarity index 88% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitMiddleware.java index 6e0cacc80..256f71bc4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitMiddleware.java @@ -1,6 +1,5 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.middleware.Middleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/Middleware.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/middleware/Middleware.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/Middleware.java index aa60d27c6..2024efa0b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/middleware/Middleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/Middleware.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.model.middleware; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; public abstract class Middleware { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/MiddlewareError.java similarity index 93% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/MiddlewareError.java index d74596f28..20850e5f0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/MiddlewareError.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.model; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; import lombok.Getter; import lombok.Setter; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java deleted file mode 100644 index 3ff8494ba..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java +++ /dev/null @@ -1,472 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.create.steps; - -import io.micronaut.context.annotation.Property; -import io.micronaut.context.annotation.Prototype; -import io.micronaut.http.server.exceptions.InternalServerException; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.core.*; -import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapi.v2.dao.*; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.ProgramLocation; -import org.breedinginsight.services.ProgramLocationService; -import org.breedinginsight.utilities.Utilities; - - - -import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; - -@Prototype -@Slf4j -public class GetExistingProcessingStep implements ProcessingStep { - - private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; - private final BrAPITrialDAO brAPITrialDAO; - private final BrAPIStudyDAO brAPIStudyDAO; - private final ProgramLocationService locationService; - private final BrAPIListDAO brAPIListDAO; - private final BrAPIGermplasmDAO brAPIGermplasmDAO; - private final StudyService studyService; - private final TrialService trialService; - - @Property(name = "brapi.server.reference-source") - private String BRAPI_REFERENCE_SOURCE; - - @Inject - public GetExistingProcessingStep(BrAPIObservationUnitDAO brAPIObservationUnitDAO, - BrAPITrialDAO brAPITrialDAO, - BrAPIStudyDAO brAPIStudyDAO, - ProgramLocationService locationService, - BrAPIListDAO brAPIListDAO, - BrAPIGermplasmDAO brAPIGermplasmDAO, - StudyService studyService, - TrialService trialService) { - this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; - this.brAPITrialDAO = brAPITrialDAO; - this.brAPIStudyDAO = brAPIStudyDAO; - this.locationService = locationService; - this.brAPIListDAO = brAPIListDAO; - this.brAPIGermplasmDAO = brAPIGermplasmDAO; - this.studyService = studyService; - this.trialService = trialService; - } - - @Override - public PendingData process(ImportContext input) { - - List experimentImportRows = ExperimentUtilities.importRowsToExperimentObservations(input.getImportRows()); - Program program = input.getProgram(); - - // Populate pending objects with existing status - Map> observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); - Map> trialByNameNoScope = initializeTrialByNameNoScope(program, observationUnitByNameNoScope, experimentImportRows); - Map> studyByNameNoScope = initializeStudyByNameNoScope(program, trialByNameNoScope, observationUnitByNameNoScope, experimentImportRows); - // interesting we're using our data model instead of brapi for locations - Map> locationByName = initializeUniqueLocationNames(program, studyByNameNoScope, experimentImportRows); - Map> obsVarDatasetByName = initializeObsVarDatasetByName(program, trialByNameNoScope, experimentImportRows); - Map> existingGermplasmByGID = initializeExistingGermplasmByGID(program, observationUnitByNameNoScope, experimentImportRows); - - PendingData existing = PendingData.builder() - .observationUnitByNameNoScope(observationUnitByNameNoScope) - .trialByNameNoScope(trialByNameNoScope) - .studyByNameNoScope(studyByNameNoScope) - .locationByName(locationByName) - .obsVarDatasetByName(obsVarDatasetByName) - .existingGermplasmByGID(existingGermplasmByGID) - .build(); - - return existing; - } - - private Map> initializeTrialByNameNoScope(Program program, Map> observationUnitByNameNoScope, List experimentImportRows) { - return null; - } - - /** - * Initializes the observation units for the given program and experimentImportRows. - * - * @param program The program object - * @param experimentImportRows A list of ExperimentObservation objects - * @return A map of Observation Unit IDs to PendingImportObject objects - * - * @throws InternalServerException - * @throws IllegalStateException - */ - private Map> initializeObservationUnits(Program program, List experimentImportRows) { - Map> observationUnitByName = new HashMap<>(); - - Map rowByObsUnitId = new HashMap<>(); - experimentImportRows.forEach(row -> { - if (StringUtils.isNotBlank(row.getObsUnitID())) { - if(rowByObsUnitId.containsKey(row.getObsUnitID())) { - throw new IllegalStateException("ObsUnitId is repeated: " + row.getObsUnitID()); - } - rowByObsUnitId.put(row.getObsUnitID(), row); - } - }); - - try { - List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); - - // TODO: grab from externalReferences - /* - observationUnitByObsUnitId = existingObsUnits.stream() - .collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, - (BrAPIObservationUnit unit) -> new PendingImportObject<>(unit, false))); - */ - - String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - if (existingObsUnits.size() == rowByObsUnitId.size()) { - existingObsUnits.forEach(brAPIObservationUnit -> { - processAndCacheObservationUnit(brAPIObservationUnit, refSource, program, observationUnitByName, rowByObsUnitId); - - BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) - .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); - - ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); - row.setExpTitle(Utilities.removeProgramKey(brAPIObservationUnit.getTrialName(), program.getKey())); - row.setEnv(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getStudyName(), program.getKey())); - row.setEnvLocation(Utilities.removeProgramKey(brAPIObservationUnit.getLocationName(), program.getKey())); - }); - } else { - List missingIds = new ArrayList<>(rowByObsUnitId.keySet()); - missingIds.removeAll(existingObsUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList())); - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); - } - - return observationUnitByName; - } catch (ApiException e) { - log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - /** - * Adds a new map entry to observationUnitByName based on the brAPIObservationUnit passed in and sets the - * expUnitId in the rowsByObsUnitId map. - * - * @param brAPIObservationUnit the BrAPI observation unit object - * @param refSource the reference source - * @param program the program object - * @param observationUnitByName the map of observation units by name (will be modified in place) - * @param rowByObsUnitId the map of rows by observation unit ID (will be modified in place) - * - * @throws InternalServerException - */ - private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservationUnit, String refSource, Program program, - Map> observationUnitByName, - Map rowByObsUnitId) { - BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) - .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); - - ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); - row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); - observationUnitByName.put(createObservationUnitKey(row), - new PendingImportObject<>(ImportObjectState.EXISTING, - brAPIObservationUnit, - UUID.fromString(idRef.getReferenceId()))); - } - - private String createObservationUnitKey(ExperimentObservation importRow) { - return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); - } - - private String createObservationUnitKey(String studyName, String obsUnitName) { - return studyName + obsUnitName; - } - - /** - * Initializes studies by name without scope. - * - * @param program The program object. - * @param trialByNameNoScope A map of trial names with their corresponding pending import objects. - * @param experimentImportRows A list of experiment observation objects. - * @return A map of study names with their corresponding pending import objects. - * @throws InternalServerException If there is an error while processing the method. - */ - private Map> initializeStudyByNameNoScope(Program program, - Map> trialByNameNoScope, - Map> observationUnitByNameNoScope, - List experimentImportRows) { - Map> studyByName = new HashMap<>(); - if (trialByNameNoScope.size() != 1) { - return studyByName; - } - - try { - initializeStudiesForExistingObservationUnits(program, studyByName, observationUnitByNameNoScope); - } catch (ApiException e) { - log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } catch (Exception e) { - log.error("Error processing studies", e); - throw new InternalServerException(e.toString(), e); - } - - List existingStudies; - Optional> trial = getTrialPIO(experimentImportRows, trialByNameNoScope); - - try { - if (trial.isEmpty()) { - // TODO: throw ValidatorException and return 422 - } - UUID experimentId = trial.get().getId(); - existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); - for (BrAPIStudy existingStudy : existingStudies) { - studyService.processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); - } - } catch (ApiException e) { - log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } catch (Exception e) { - log.error("Error processing studies: ", e); - throw new InternalServerException(e.toString(), e); - } - - return studyByName; - } - - /** - * Retrieves the PendingImportObject of a BrAPITrial based on the given list of ExperimentObservation and trialByNameNoScope map. - * - * @param experimentImportRows The list of ExperimentObservation objects. - * @param trialByNameNoScope The map of trial names to PendingImportObject of BrAPITrial. - * @return The Optional containing the PendingImportObject of BrAPITrial, or an empty Optional if no matching trial is found. - */ - private Optional> getTrialPIO(List experimentImportRows, - Map> trialByNameNoScope) { - Optional expTitle = experimentImportRows.stream() - .filter(row -> StringUtils.isBlank(row.getObsUnitID()) && StringUtils.isNotBlank(row.getExpTitle())) - .map(ExperimentObservation::getExpTitle) - .findFirst(); - - if (expTitle.isEmpty() && trialByNameNoScope.keySet().stream().findFirst().isEmpty()) { - return Optional.empty(); - } - if(expTitle.isEmpty()) { - expTitle = trialByNameNoScope.keySet().stream().findFirst(); - } - - return Optional.ofNullable(trialByNameNoScope.get(expTitle.get())); - } - - - private void initializeStudiesForExistingObservationUnits( - Program program, - Map> studyByName, - Map> observationUnitByNameNoScope - ) throws Exception { - Set studyDbIds = observationUnitByNameNoScope.values() - .stream() - .map(pio -> pio.getBrAPIObject() - .getStudyDbId()) - .collect(Collectors.toSet()); - - List studies = fetchStudiesByDbId(studyDbIds, program); - for (BrAPIStudy study : studies) { - studyService.processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); - } - } - - private List fetchStudiesByDbId(Set studyDbIds, Program program) { - return null; - } - - /** - * Initializes unique location names for a program. - * - * @param program The program object. - * @param studyByNameNoScope A map of study names and corresponding BrAPI study objects. - * @param experimentImportRows A list of experiment observation objects for import. - * @return A map of location names and their corresponding pending import objects. - * @throws InternalServerException If there is an error fetching locations. - */ - private Map> initializeUniqueLocationNames(Program program, - Map> studyByNameNoScope, - List experimentImportRows) { - Map> locationByName = new HashMap<>(); - - List existingLocations = new ArrayList<>(); - if(studyByNameNoScope.size() > 0) { - Set locationDbIds = studyByNameNoScope.values() - .stream() - .map(study -> study.getBrAPIObject() - .getLocationDbId()) - .collect(Collectors.toSet()); - try { - existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); - } catch (ApiException e) { - log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - List uniqueLocationNames = experimentImportRows.stream() - .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) - .map(ExperimentObservation::getEnvLocation) - .distinct() - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - try { - existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); - } catch (ApiException e) { - log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - - existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); - return locationByName; - } - - /** - * Initializes observation variable dataset by name. - * - * @param program The program associated with the dataset. - * @param trialByNameNoScope The map of trials identified by name without scope. - * @param experimentImportRows The list of experiment observation rows. - * @return The map of observation variable dataset indexed by name. - * - * @throws InternalServerException - */ - private Map> initializeObsVarDatasetByName(Program program, - Map> trialByNameNoScope, - List experimentImportRows) { - Map> obsVarDatasetByName = new HashMap<>(); - - Optional> trialPIO = getTrialPIO(experimentImportRows, trialByNameNoScope); - - if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { - String datasetId = trialPIO.get().getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) - .getAsString(); - try { - List existingDatasets = brAPIListDAO - .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, - program.getId(), - String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), - UUID.fromString(datasetId)); - if (existingDatasets == null || existingDatasets.isEmpty()) { - throw new InternalServerException("existing dataset summary not returned from brapi server"); - } - BrAPIListDetails dataSetDetails = brAPIListDAO - .getListById(existingDatasets.get(0).getListDbId(), program.getId()) - .getResult(); - processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); - } catch (ApiException e) { - log.error(Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - return obsVarDatasetByName; - } - - /** - * Process and cache an object of type BrAPIListDetails. - * - * @param existingList The existing list to be processed and cached - * @param obsVarDatasetByName A map of ObsVarDatasets indexed by name (will be modified in place) - * - * @throws IllegalStateException - */ - private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map> obsVarDatasetByName) { - BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), - String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) - .orElseThrow(() -> new IllegalStateException("External references wasn't found for list (dbid): " + existingList.getListDbId())); - obsVarDatasetByName.put(existingList.getListName(), - new PendingImportObject<>(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceId()))); - } - - /** - * Initializes existing germplasm objects by germplasm ID (GID). - * - * @param program The program object. - * @param observationUnitByNameNoScope A map of observation unit objects by name. - * @param experimentImportRows A list of experiment observation objects. - * @return A map of existing germplasm objects by germplasm ID. - * - * @throws InternalServerException - */ - private Map> initializeExistingGermplasmByGID(Program program, - Map> observationUnitByNameNoScope, - List experimentImportRows) { - Map> existingGermplasmByGID = new HashMap<>(); - - List existingGermplasms = new ArrayList<>(); - if(observationUnitByNameNoScope.size() > 0) { - Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); - try { - existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); - } catch (ApiException e) { - log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - List uniqueGermplasmGIDs = experimentImportRows.stream() - .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) - .map(ExperimentObservation::getGid) - .distinct() - .collect(Collectors.toList()); - - try { - existingGermplasms.addAll(getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); - } catch (ApiException e) { - log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - - existingGermplasms.forEach(existingGermplasm -> { - BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) - .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); - existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); - }); - return existingGermplasmByGID; - } - - /** - * Retrieves a list of germplasm with the specified accession numbers. - * - * @param germplasmAccessionNumbers The list of accession numbers to search for. - * @param programId The ID of the program. - * @return An ArrayList of BrAPIGermplasm objects that match the accession numbers. - * @throws ApiException if there is an error retrieving the germplasm. - */ - private ArrayList getGermplasmByAccessionNumber( - List germplasmAccessionNumbers, - UUID programId) throws ApiException { - List germplasmList = brAPIGermplasmDAO.getGermplasm(programId); - ArrayList resultGermplasm = new ArrayList<>(); - // Search for accession number matches - for (BrAPIGermplasm germplasm : germplasmList) { - for (String accessionNumber : germplasmAccessionNumbers) { - if (germplasm.getAccessionNumber() - .equals(accessionNumber)) { - resultGermplasm.add(germplasm); - break; - } - } - } - return resultGermplasm; - } - -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java deleted file mode 100644 index cfab4b5a1..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.create.steps; - -import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; - -import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; - -public class ProcessStep implements ProcessingStep { - - @Override - public ProcessedData process(PendingData input) { - - // TODO: implement - return new ProcessedData(); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java index f0e55b57c..992c44a32 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java @@ -1,36 +1,18 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.service; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; import io.micronaut.http.HttpStatus; -import io.micronaut.http.server.exceptions.InternalServerException; -import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.api.model.v1.response.ValidationError; -import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.ProcessorData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.OverwrittenData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; import org.breedinginsight.services.OntologyService; import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.utilities.Utilities; import tech.tablesaw.columns.Column; import org.breedinginsight.dao.db.tables.pojos.TraitEntity; import javax.inject.Inject; import javax.inject.Singleton; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; import java.util.*; import java.util.stream.Collectors; From 5567db6489963f4ab5d7ec42103418239f81c9dd Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 21 Jun 2024 20:14:22 -0400 Subject: [PATCH 156/203] move expunitmiddleware context to model package --- .../services/processors/experiment/ExperimentUtilities.java | 2 +- .../appendoverwrite/AppendOverwritePhenotypesWorkflow.java | 2 +- .../appendoverwrite/factory/action/BrAPICreationFactory.java | 2 +- .../appendoverwrite/factory/action/BrAPIReadFactory.java | 2 +- .../appendoverwrite/factory/action/BrAPIUpdateFactory.java | 2 +- .../appendoverwrite/factory/entity/PendingDataset.java | 2 +- .../appendoverwrite/factory/entity/PendingEntityFactory.java | 2 +- .../appendoverwrite/factory/entity/PendingGermplasm.java | 2 +- .../appendoverwrite/factory/entity/PendingLocation.java | 2 +- .../appendoverwrite/factory/entity/PendingObservation.java | 2 +- .../factory/entity/PendingObservationUnit.java | 2 +- .../appendoverwrite/factory/entity/PendingStudy.java | 2 +- .../appendoverwrite/factory/entity/PendingTrial.java | 2 +- .../appendoverwrite/middleware/ExpUnitIDValidation.java | 2 +- .../appendoverwrite/middleware/GetExistingBrAPIData.java | 2 +- .../experiment/appendoverwrite/middleware/Transaction.java | 2 +- .../appendoverwrite/middleware/commit/BrAPICommit.java | 2 +- .../appendoverwrite/middleware/commit/BrAPIDatasetCommit.java | 2 +- .../middleware/commit/BrAPIObservationCommit.java | 2 +- .../middleware/commit/BrAPIObservationUnitCommit.java | 2 +- .../appendoverwrite/middleware/commit/BrAPIStudyCommit.java | 2 +- .../appendoverwrite/middleware/commit/BrAPITrialCommit.java | 2 +- .../appendoverwrite/middleware/commit/LocationCommit.java | 2 +- .../middleware/initialize/WorkflowInitialization.java | 2 +- .../middleware/process/ImportTableProcess.java | 2 +- .../experiment/appendoverwrite/model/ExpUnitMiddleware.java | 2 -- .../{ => appendoverwrite}/model/ExpUnitMiddlewareContext.java | 4 +++- 27 files changed, 28 insertions(+), 28 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ => appendoverwrite}/model/ExpUnitMiddlewareContext.java (80%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index f807095d3..2bff28f7f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -16,7 +16,7 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.UnprocessableEntityException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 553f70316..f1f95efa1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -17,7 +17,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize.WorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.ImportTableProcess; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java index fdfee9a5e..0cf05d369 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java @@ -9,7 +9,7 @@ import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.PendingEntityFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.model.ProgramLocation; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java index 5470c557f..69bce3c31 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java @@ -9,7 +9,7 @@ import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.PendingEntityFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.model.ProgramLocation; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java index f8796f708..cafb1db51 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java @@ -7,7 +7,7 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.PendingEntityFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java index b628ecf4d..88508a265 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java @@ -11,7 +11,7 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; import org.breedinginsight.services.exceptions.DoesNotExistException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java index b65ee97f2..26e4630cc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java @@ -5,7 +5,7 @@ import io.micronaut.context.annotation.Prototype; import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.*; import org.breedinginsight.services.OntologyService; import org.breedinginsight.services.ProgramLocationService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java index f0e3daff5..0db9d5219 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java @@ -8,7 +8,7 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.GermplasmService; import org.breedinginsight.services.exceptions.DoesNotExistException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java index 08ba57159..d0807b5e5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java @@ -9,7 +9,7 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; import org.breedinginsight.model.ProgramLocation; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java index efc675a7b..a47157325 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java @@ -8,7 +8,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Trait; import org.breedinginsight.services.OntologyService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java index d197acfc6..75e107956 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java @@ -10,7 +10,7 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; import org.breedinginsight.services.exceptions.DoesNotExistException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java index ee85b283e..fa614b4dd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java @@ -8,7 +8,7 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; import org.breedinginsight.services.exceptions.DoesNotExistException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java index a34e9d349..e395c338a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java @@ -8,7 +8,7 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; import org.breedinginsight.utilities.Utilities; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java index 6a945692c..90879f23c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java @@ -4,7 +4,7 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; @Slf4j @Prototype diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java index f48706375..c46a89a2d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java @@ -11,7 +11,7 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java index 7d13227da..1bd68fb3d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java @@ -3,7 +3,7 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; @Slf4j @Prototype diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java index 7b7a964fc..c20d494cf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java @@ -3,7 +3,7 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index 5a677a993..7d0c6ba96 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -9,7 +9,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java index bb583dba5..9881d13e1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -9,7 +9,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index 8b1b842b4..c2f74dab3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -7,7 +7,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java index ccf47edfc..8cc62023e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -7,7 +7,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index d9e41f7be..a4dcd5a4b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -9,7 +9,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index 1438d0fe8..4b05c7246 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -6,7 +6,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import org.breedinginsight.model.ProgramLocation; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index e9b0230a2..e9b141121 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -11,7 +11,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIReadFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowReadInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import org.breedinginsight.model.ProgramLocation; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index c59dc0ebe..8fe06dca1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -27,7 +27,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.VisitedObservationData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationVariableService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitMiddleware.java index 256f71bc4..9fb5c04da 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitMiddleware.java @@ -1,7 +1,5 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; - /** * ExpUnitMiddleware class extends Middleware class to handle compensating transactions in the context of ExpUnitMiddlewareContext. */ diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitMiddlewareContext.java similarity index 80% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitMiddlewareContext.java index f50c82ad9..dc64e172b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitMiddlewareContext.java @@ -1,10 +1,12 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.model; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; import lombok.Builder; import lombok.Getter; import lombok.Setter; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; + @Getter @Setter @Builder From 98d4b7d64439c2e140b3d9aa2aa1fed8818f2824 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 21 Jun 2024 20:29:41 -0400 Subject: [PATCH 157/203] clean up context models --- .../AppendOverwritePhenotypesWorkflow.java | 4 ++-- .../factory/entity/PendingDataset.java | 4 ++-- .../factory/entity/PendingGermplasm.java | 4 ++-- .../factory/entity/PendingLocation.java | 4 ++-- .../factory/entity/PendingObservation.java | 4 ++-- .../entity/PendingObservationUnit.java | 4 ++-- .../factory/entity/PendingStudy.java | 4 ++-- .../factory/entity/PendingTrial.java | 4 ++-- ...ontext.java => AppendWorkflowContext.java} | 9 ++++++--- .../model/ExpUnitMiddlewareContext.java | 6 +----- .../experiment/service/LocationService.java | 8 -------- .../service/ObservationUnitService.java | 13 ------------ .../experiment/service/StudyService.java | 8 -------- .../experiment/service/TrialService.java | 4 ---- .../experiment/service/ValidateService.java | 20 ------------------- 15 files changed, 23 insertions(+), 77 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/{ExpUnitContext.java => AppendWorkflowContext.java} (93%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index f1f95efa1..53df60afd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -16,7 +16,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.BrAPICommit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize.WorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.ImportTableProcess; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; @@ -85,7 +85,7 @@ public Optional process(ImportServiceContext context) { .build(); ExpUnitMiddlewareContext workflowContext = ExpUnitMiddlewareContext.builder() .importContext(importContext) - .expUnitContext(new ExpUnitContext()) + .expUnitContext(new AppendWorkflowContext()) .build(); // Process the import preview diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java index 88508a265..febafde41 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java @@ -10,7 +10,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; @@ -25,7 +25,7 @@ @Prototype public class PendingDataset implements ExperimentImportEntity { - ExpUnitContext cache; + AppendWorkflowContext cache; ImportContext importContext; BrAPIListDAO brAPIListDAO; DatasetService datasetService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java index 0db9d5219..2755a21ff 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java @@ -7,7 +7,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.GermplasmService; @@ -20,7 +20,7 @@ @Prototype public class PendingGermplasm implements ExperimentImportEntity { - ExpUnitContext cache; + AppendWorkflowContext cache; ImportContext importContext; GermplasmService germplasmService; ExperimentUtilities experimentUtilities; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java index d0807b5e5..3dacbbc2f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java @@ -8,7 +8,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; @@ -23,7 +23,7 @@ @Prototype public class PendingLocation implements ExperimentImportEntity { - ExpUnitContext cache; + AppendWorkflowContext cache; ImportContext importContext; ProgramLocationService programLocationService; LocationService locationService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java index a47157325..76a3b7e12 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java @@ -7,7 +7,7 @@ import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Trait; @@ -21,7 +21,7 @@ @Prototype public class PendingObservation implements ExperimentImportEntity { - ExpUnitContext cache; + AppendWorkflowContext cache; ImportContext importContext; BrAPIObservationDAO brAPIObservationDAO; OntologyService ontologyService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java index 75e107956..240ece9cc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java @@ -9,7 +9,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; @@ -23,7 +23,7 @@ @Prototype public class PendingObservationUnit implements ExperimentImportEntity { - ExpUnitContext cache; + AppendWorkflowContext cache; ImportContext importContext; BrAPIObservationUnitDAO observationUnitDAO; ObservationUnitService observationUnitService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java index fa614b4dd..6b2e001d6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java @@ -7,7 +7,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; @@ -21,7 +21,7 @@ @Prototype public class PendingStudy implements ExperimentImportEntity{ - ExpUnitContext cache; + AppendWorkflowContext cache; ImportContext importContext; StudyService studyService; BrAPIStudyDAO brAPIStudyDAO; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java index e395c338a..27a339339 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java @@ -7,7 +7,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; @@ -18,7 +18,7 @@ @Prototype public class PendingTrial implements ExperimentImportEntity { - private final ExpUnitContext cache; + private final AppendWorkflowContext cache; private final ImportContext importContext; private final TrialService trialService; private final BrAPITrialDAO brapiTrialDAO; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendWorkflowContext.java similarity index 93% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendWorkflowContext.java index 41990c8e1..0584cc71e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendWorkflowContext.java @@ -21,7 +21,8 @@ @Getter @Setter -public class ExpUnitContext { +public class AppendWorkflowContext { + // Cache maps keyed by existing observation unit ids private Set referenceOUIds = new HashSet<>(); private Map> pendingTrialByOUId = new HashMap<>(); private Map> pendingStudyByOUId = new HashMap<>(); @@ -35,16 +36,18 @@ public class ExpUnitContext { // Exceptions private MiddlewareError processError; + private ValidationErrors validationErrors; - // Carry over from PendingData + // Cache maps keyed by name without program scope private Map> observationUnitByNameNoScope; private Map> trialByNameNoScope; private Map> studyByNameNoScope; private Map> locationByName; private Map> obsVarDatasetByName; + + // Other helpful cache maps private Map> existingGermplasmByGID; private Map> pendingObservationByHash; private Map> timeStampColByPheno; private Map existingObsByObsHash; - private ValidationErrors validationErrors; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitMiddlewareContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitMiddlewareContext.java index dc64e172b..b589fdc8f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitMiddlewareContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitMiddlewareContext.java @@ -3,8 +3,6 @@ import lombok.Builder; import lombok.Getter; import lombok.Setter; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; @Getter @@ -13,7 +11,5 @@ public class ExpUnitMiddlewareContext { private ImportContext importContext; - private ExpUnitContext expUnitContext; - private PendingData pendingData; - + private AppendWorkflowContext expUnitContext; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java index faca12734..3305545c0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java @@ -2,19 +2,11 @@ import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.api.auth.AuthenticatedUser; -import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.ProcessorData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.services.ProgramLocationService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index 489c36f0a..1b1e3479c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -1,29 +1,16 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.context.annotation.Property; -import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.api.model.v1.response.ValidationErrors; -import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; -import org.breedinginsight.model.ProgramLocation; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index 4304cbc74..fb21be4cc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -8,29 +8,21 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPISeason; import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; -import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; -import org.breedinginsight.model.ProgramLocation; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; -import java.math.BigInteger; import java.util.*; -import java.util.function.Supplier; import java.util.stream.Collectors; import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index fe68e61c3..119c67ed4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -14,14 +14,10 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; -import org.breedinginsight.model.ProgramLocation; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java index ccdae7937..582eb78bf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java @@ -1,25 +1,5 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.service; -import org.apache.commons.collections4.map.CaseInsensitiveMap; -import org.apache.commons.lang3.StringUtils; -import org.brapi.v2.model.pheno.BrAPIObservation; -import org.breedinginsight.api.model.v1.response.ValidationErrors; -import org.breedinginsight.brapps.importer.model.imports.PendingImport; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.Trait; -import org.breedinginsight.utilities.Utilities; -import tech.tablesaw.columns.Column; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - public class ValidateService { // TODO: used by expUnit workflow // public void prepareDataForValidation(ImportContext importContext, From 2cc3f0926abbcec00ae87816759e90ee127a1694 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Sun, 23 Jun 2024 14:41:07 -0400 Subject: [PATCH 158/203] rename context models --- .../experiment/ExperimentUtilities.java | 4 +- .../AppendOverwritePhenotypesWorkflow.java | 28 ++++----- .../factory/action/BrAPICreationFactory.java | 26 ++++----- .../factory/action/BrAPIReadFactory.java | 30 +++++----- .../factory/action/BrAPIUpdateFactory.java | 16 ++--- .../factory/entity/PendingDataset.java | 10 ++-- .../factory/entity/PendingEntityFactory.java | 30 +++++----- .../factory/entity/PendingGermplasm.java | 10 ++-- .../factory/entity/PendingLocation.java | 10 ++-- .../factory/entity/PendingObservation.java | 10 ++-- .../entity/PendingObservationUnit.java | 10 ++-- .../factory/entity/PendingStudy.java | 10 ++-- .../factory/entity/PendingTrial.java | 10 ++-- ....java => AppendOverwriteIDValidation.java} | 14 ++--- .../middleware/GetExistingBrAPIData.java | 20 +++---- .../middleware/Transaction.java | 10 ++-- .../middleware/commit/BrAPICommit.java | 12 ++-- .../middleware/commit/BrAPIDatasetCommit.java | 14 ++--- .../commit/BrAPIObservationCommit.java | 14 ++--- .../commit/BrAPIObservationUnitCommit.java | 14 ++--- .../middleware/commit/BrAPIStudyCommit.java | 14 ++--- .../middleware/commit/BrAPITrialCommit.java | 14 ++--- .../middleware/commit/LocationCommit.java | 14 ++--- .../initialize/WorkflowInitialization.java | 10 ++-- .../process/ImportTableProcess.java | 58 +++++++++---------- ...re.java => AppendOverwriteMiddleware.java} | 6 +- ... => AppendOverwriteMiddlewareContext.java} | 4 +- ...va => AppendOverwriteWorkflowContext.java} | 2 +- 28 files changed, 212 insertions(+), 212 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{ExpUnitIDValidation.java => AppendOverwriteIDValidation.java} (51%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/{ExpUnitMiddleware.java => AppendOverwriteMiddleware.java} (71%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/{ExpUnitMiddlewareContext.java => AppendOverwriteMiddlewareContext.java} (72%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/{AppendWorkflowContext.java => AppendOverwriteWorkflowContext.java} (98%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 2bff28f7f..30eadea28 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -15,8 +15,8 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.UnprocessableEntityException; @@ -144,7 +144,7 @@ public void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study, Stri } } - public static Set collateReferenceOUIds(ExpUnitMiddlewareContext context) { + public static Set collateReferenceOUIds(AppendOverwriteMiddlewareContext context) { Set referenceOUIds = new HashSet<>(); boolean hasNoReferenceUnitIds = true; boolean hasAllReferenceUnitIds = true; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 53df60afd..308821f31 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -10,14 +10,14 @@ import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; import org.breedinginsight.brapps.importer.services.ImportStatusService; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.AppendOverwriteIDValidation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.Transaction; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitIDValidation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.BrAPICommit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize.WorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.ImportTableProcess; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendWorkflowContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import javax.inject.Inject; @@ -29,25 +29,25 @@ @Singleton public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow { private final ExperimentWorkflowNavigator.Workflow workflow; - private final ExpUnitMiddleware importPreviewMiddleware; - private final ExpUnitMiddleware brapiCommitMiddleware; + private final AppendOverwriteMiddleware importPreviewMiddleware; + private final AppendOverwriteMiddleware brapiCommitMiddleware; private final ImportStatusService statusService; @Inject public AppendOverwritePhenotypesWorkflow(Transaction transaction, - ExpUnitIDValidation expUnitIDValidation, + AppendOverwriteIDValidation expUnitIDValidation, WorkflowInitialization workflowInitialization, ImportTableProcess importTableProcess, BrAPICommit brAPICommit, ImportStatusService statusService){ this.statusService = statusService; this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE; - this.importPreviewMiddleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( + this.importPreviewMiddleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link( transaction, expUnitIDValidation, workflowInitialization, importTableProcess); - this.brapiCommitMiddleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(brAPICommit); + this.brapiCommitMiddleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link(brAPICommit); } @Override @@ -83,13 +83,13 @@ public Optional process(ImportServiceContext context) { .user(context.getUser()) .commit(context.isCommit()) .build(); - ExpUnitMiddlewareContext workflowContext = ExpUnitMiddlewareContext.builder() + AppendOverwriteMiddlewareContext workflowContext = AppendOverwriteMiddlewareContext.builder() .importContext(importContext) - .expUnitContext(new AppendWorkflowContext()) + .appendOverwriteWorkflowContext(new AppendOverwriteWorkflowContext()) .build(); // Process the import preview - ExpUnitMiddlewareContext processedPreviewContext = this.importPreviewMiddleware.process(workflowContext); + AppendOverwriteMiddlewareContext processedPreviewContext = this.importPreviewMiddleware.process(workflowContext); // TODO: Rethrow any exceptions caught during processing the context // Optional.ofNullable(processedContext.getExpUnitContext().getMiddlewareError()).ifPresent(e -> { @@ -100,7 +100,7 @@ public Optional process(ImportServiceContext context) { // BUild and return the preview response ImportPreviewResponse response = new ImportPreviewResponse(); - response.setStatistics(processedPreviewContext.getExpUnitContext().getStatistic().constructPreviewMap()); + response.setStatistics(processedPreviewContext.getAppendOverwriteWorkflowContext().getStatistic().constructPreviewMap()); response.setRows(new ArrayList<>(processedPreviewContext.getImportContext().getMappedBrAPIImport().values())); response.setDynamicColumnNames(processedPreviewContext.getImportContext().getUpload().getDynamicColumnNamesList()); @@ -124,7 +124,7 @@ public Optional process(ImportServiceContext context) { statusService.updateMessage(context.getUpload(), "Creating new objects in brapi service"); // Commit the changes from the processed import preview to the BrAPI service - ExpUnitMiddlewareContext brapiCommittedContext = this.brapiCommitMiddleware.process(processedPreviewContext); + AppendOverwriteMiddlewareContext brapiCommittedContext = this.brapiCommitMiddleware.process(processedPreviewContext); log.debug("Completed upload to brapi service"); statusService.finishUpload(context.getUpload(), totalObjects, "Completed upload to brapi service"); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java index 0cf05d369..fdbd41d25 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java @@ -9,7 +9,7 @@ import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.PendingEntityFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.model.ProgramLocation; import javax.inject.Inject; @@ -23,69 +23,69 @@ public BrAPICreationFactory(PendingEntityFactory pendingEntityFactory) { this.pendingEntityFactory = pendingEntityFactory; } - public static WorkflowCreation trialWorkflowCreation(ExpUnitMiddlewareContext context, + public static WorkflowCreation trialWorkflowCreation(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowCreation(pendingEntityFactory.pendingTrialBean(context)); } - public static WorkflowCreation datasetWorkflowCreation(ExpUnitMiddlewareContext context, + public static WorkflowCreation datasetWorkflowCreation(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowCreation(pendingEntityFactory.pendingDatasetBean(context)); } - public static WorkflowCreation studyWorkflowCreation(ExpUnitMiddlewareContext context, + public static WorkflowCreation studyWorkflowCreation(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowCreation(pendingEntityFactory.pendingStudyBean(context)); } - public static WorkflowCreation observationWorkflowCreation(ExpUnitMiddlewareContext context, + public static WorkflowCreation observationWorkflowCreation(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowCreation(pendingEntityFactory.pendingObservationBean(context)); } - public static WorkflowCreation observationUnitWorkflowCreation(ExpUnitMiddlewareContext context, + public static WorkflowCreation observationUnitWorkflowCreation(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowCreation(pendingEntityFactory.pendingObservationUnitBean(context)); } - public static WorkflowCreation locationWorkflowCreation(ExpUnitMiddlewareContext context, + public static WorkflowCreation locationWorkflowCreation(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowCreation(pendingEntityFactory.pendingLocationBean(context)); } @Bean @Prototype - public WorkflowCreation trialWorkflowCreationBean(ExpUnitMiddlewareContext context) { + public WorkflowCreation trialWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { return trialWorkflowCreation(context, pendingEntityFactory); } @Bean @Prototype - public WorkflowCreation datasetWorkflowCreationBean(ExpUnitMiddlewareContext context) { + public WorkflowCreation datasetWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { return datasetWorkflowCreation(context, pendingEntityFactory); } @Bean @Prototype - public WorkflowCreation studyWorkflowCreationBean(ExpUnitMiddlewareContext context) { + public WorkflowCreation studyWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { return studyWorkflowCreation(context, pendingEntityFactory); } @Bean @Prototype - public WorkflowCreation observationWorkflowCreationBean(ExpUnitMiddlewareContext context) { + public WorkflowCreation observationWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { return observationWorkflowCreation(context, pendingEntityFactory); } @Bean @Prototype - public WorkflowCreation observationUnitWorkflowCreationBean(ExpUnitMiddlewareContext context) { + public WorkflowCreation observationUnitWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { return observationUnitWorkflowCreation(context, pendingEntityFactory); } @Bean @Prototype - public WorkflowCreation locationWorkflowCreationBean(ExpUnitMiddlewareContext context) { + public WorkflowCreation locationWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { return locationWorkflowCreation(context, pendingEntityFactory); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java index 69bce3c31..30a9ae341 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java @@ -9,7 +9,7 @@ import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.PendingEntityFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.model.ProgramLocation; import javax.inject.Inject; @@ -23,69 +23,69 @@ public BrAPIReadFactory(PendingEntityFactory pendingEntityFactory) { this.pendingEntityFactory = pendingEntityFactory; } - public static WorkflowReadInitialization trialWorkflowReadInitialization(ExpUnitMiddlewareContext context, + public static WorkflowReadInitialization trialWorkflowReadInitialization(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowReadInitialization(pendingEntityFactory.pendingTrialBean(context)); } - public static WorkflowReadInitialization observationUnitWorkflowReadInitialization(ExpUnitMiddlewareContext context, + public static WorkflowReadInitialization observationUnitWorkflowReadInitialization(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowReadInitialization(pendingEntityFactory.pendingObservationUnitBean(context)); } - public static WorkflowReadInitialization germplasmWorkflowReadInitialization(ExpUnitMiddlewareContext context, + public static WorkflowReadInitialization germplasmWorkflowReadInitialization(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowReadInitialization(pendingEntityFactory.pendingGermplasmBean(context)); } - public static WorkflowReadInitialization datasetWorkflowReadInitialization(ExpUnitMiddlewareContext context, - PendingEntityFactory pendingEntityFactory) { + public static WorkflowReadInitialization datasetWorkflowReadInitialization(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { return new WorkflowReadInitialization(pendingEntityFactory.pendingDatasetBean(context)); } - public static WorkflowReadInitialization studyWorkflowReadInitialization(ExpUnitMiddlewareContext context, + public static WorkflowReadInitialization studyWorkflowReadInitialization(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowReadInitialization(pendingEntityFactory.pendingStudyBean(context)); } - public static WorkflowReadInitialization locationWorkflowReadInitialization(ExpUnitMiddlewareContext context, - PendingEntityFactory pendingEntityFactory) { + public static WorkflowReadInitialization locationWorkflowReadInitialization(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { return new WorkflowReadInitialization(pendingEntityFactory.pendingLocationBean(context)); } @Bean @Prototype - public WorkflowReadInitialization trialWorkflowReadInitializationBean(ExpUnitMiddlewareContext context) { + public WorkflowReadInitialization trialWorkflowReadInitializationBean(AppendOverwriteMiddlewareContext context) { return trialWorkflowReadInitialization(context, pendingEntityFactory); } @Bean @Prototype - public WorkflowReadInitialization observationUnitWorkflowReadInitializationBean(ExpUnitMiddlewareContext context) { + public WorkflowReadInitialization observationUnitWorkflowReadInitializationBean(AppendOverwriteMiddlewareContext context) { return observationUnitWorkflowReadInitialization(context, pendingEntityFactory); } @Bean @Prototype - public WorkflowReadInitialization germplasmWorkflowReadInitializationBean(ExpUnitMiddlewareContext context) { + public WorkflowReadInitialization germplasmWorkflowReadInitializationBean(AppendOverwriteMiddlewareContext context) { return germplasmWorkflowReadInitialization(context, pendingEntityFactory); } @Bean @Prototype - public WorkflowReadInitialization datasetWorkflowReadInitializationBean(ExpUnitMiddlewareContext context) { + public WorkflowReadInitialization datasetWorkflowReadInitializationBean(AppendOverwriteMiddlewareContext context) { return datasetWorkflowReadInitialization(context, pendingEntityFactory); } @Bean @Prototype - public WorkflowReadInitialization studyWorkflowReadInitializationBean(ExpUnitMiddlewareContext context) { + public WorkflowReadInitialization studyWorkflowReadInitializationBean(AppendOverwriteMiddlewareContext context) { return studyWorkflowReadInitialization(context, pendingEntityFactory); } @Bean @Prototype - public WorkflowReadInitialization locationWorkflowReadInitializationBean(ExpUnitMiddlewareContext context) { + public WorkflowReadInitialization locationWorkflowReadInitializationBean(AppendOverwriteMiddlewareContext context) { return locationWorkflowReadInitialization(context, pendingEntityFactory); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java index cafb1db51..7e794b868 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java @@ -7,7 +7,7 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.PendingEntityFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import javax.inject.Inject; @@ -20,36 +20,36 @@ public BrAPIUpdateFactory(PendingEntityFactory pendingEntityFactory) { this.pendingEntityFactory = pendingEntityFactory; } - public static WorkflowUpdate trialWorkflowUpdate(ExpUnitMiddlewareContext context, + public static WorkflowUpdate trialWorkflowUpdate(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowUpdate(pendingEntityFactory.pendingTrialBean(context)); } - public static WorkflowUpdate observationWorkflowUpdate(ExpUnitMiddlewareContext context, - PendingEntityFactory pendingEntityFactory) { + public static WorkflowUpdate observationWorkflowUpdate(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { return new WorkflowUpdate(pendingEntityFactory.pendingObservationBean(context)); } - public static WorkflowUpdate datasetWorkflowUpdate(ExpUnitMiddlewareContext context, + public static WorkflowUpdate datasetWorkflowUpdate(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowUpdate(pendingEntityFactory.pendingDatasetBean(context)); } @Bean @Prototype - public WorkflowUpdate trialWorkflowUpdateBean(ExpUnitMiddlewareContext context) { + public WorkflowUpdate trialWorkflowUpdateBean(AppendOverwriteMiddlewareContext context) { return trialWorkflowUpdate(context, pendingEntityFactory); } @Bean @Prototype - public WorkflowUpdate observationWorkflowUpdateBean(ExpUnitMiddlewareContext context) { + public WorkflowUpdate observationWorkflowUpdateBean(AppendOverwriteMiddlewareContext context) { return observationWorkflowUpdate(context, pendingEntityFactory); } @Bean @Prototype - public WorkflowUpdate datasetWorkflowUpdateBean(ExpUnitMiddlewareContext context) { + public WorkflowUpdate datasetWorkflowUpdateBean(AppendOverwriteMiddlewareContext context) { return datasetWorkflowUpdate(context, pendingEntityFactory); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java index febafde41..2694e8974 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java @@ -10,8 +10,8 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendWorkflowContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; import org.breedinginsight.services.exceptions.DoesNotExistException; @@ -25,17 +25,17 @@ @Prototype public class PendingDataset implements ExperimentImportEntity { - AppendWorkflowContext cache; + AppendOverwriteWorkflowContext cache; ImportContext importContext; BrAPIListDAO brAPIListDAO; DatasetService datasetService; ExperimentUtilities experimentUtilities; - public PendingDataset(ExpUnitMiddlewareContext context, + public PendingDataset(AppendOverwriteMiddlewareContext context, BrAPIListDAO brAPIListDAO, DatasetService datasetService, ExperimentUtilities experimentUtilities) { - this.cache = context.getExpUnitContext(); + this.cache = context.getAppendOverwriteWorkflowContext(); this.importContext = context.getImportContext(); this.brAPIListDAO = brAPIListDAO; this.datasetService = datasetService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java index 26e4630cc..9823d1f3c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java @@ -5,7 +5,7 @@ import io.micronaut.context.annotation.Prototype; import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.*; import org.breedinginsight.services.OntologyService; import org.breedinginsight.services.ProgramLocationService; @@ -58,48 +58,48 @@ public PendingEntityFactory(TrialService trialService, this.experimentUtilities = experimentUtilities; } - public static PendingTrial pendingTrial(ExpUnitMiddlewareContext context, + public static PendingTrial pendingTrial(AppendOverwriteMiddlewareContext context, TrialService trialService, BrAPITrialDAO brapiTrialDAO, ExperimentUtilities experimentUtilities) { return new PendingTrial(context, trialService, brapiTrialDAO, experimentUtilities); } - public static PendingObservationUnit pendingObservationUnit(ExpUnitMiddlewareContext context, + public static PendingObservationUnit pendingObservationUnit(AppendOverwriteMiddlewareContext context, BrAPIObservationUnitDAO observationUnitDAO, ObservationUnitService observationUnitService, ExperimentUtilities experimentUtilities) { return new PendingObservationUnit(context, observationUnitDAO, observationUnitService, experimentUtilities); } - public static PendingStudy pendingStudy(ExpUnitMiddlewareContext context, + public static PendingStudy pendingStudy(AppendOverwriteMiddlewareContext context, StudyService studyService, BrAPIStudyDAO brAPIStudyDAO, ExperimentUtilities experimentUtilities) { return new PendingStudy(context, studyService, brAPIStudyDAO, experimentUtilities); } - public static PendingGermplasm pendingGermplasm(ExpUnitMiddlewareContext context, + public static PendingGermplasm pendingGermplasm(AppendOverwriteMiddlewareContext context, GermplasmService germplasmService, ExperimentUtilities experimentUtilities) { return new PendingGermplasm(context, germplasmService, experimentUtilities); } - public static PendingDataset pendingDataset(ExpUnitMiddlewareContext context, + public static PendingDataset pendingDataset(AppendOverwriteMiddlewareContext context, BrAPIListDAO brAPIListDAO, DatasetService datasetService, ExperimentUtilities experimentUtilities) { return new PendingDataset(context, brAPIListDAO, datasetService, experimentUtilities); } - public static PendingObservation pendingObservation(ExpUnitMiddlewareContext context, + public static PendingObservation pendingObservation(AppendOverwriteMiddlewareContext context, BrAPIObservationDAO brAPIObservationDAO, OntologyService ontologyService, ExperimentUtilities experimentUtilities) { return new PendingObservation(context, brAPIObservationDAO, ontologyService, experimentUtilities); } - public static PendingLocation pendingLocation(ExpUnitMiddlewareContext context, + public static PendingLocation pendingLocation(AppendOverwriteMiddlewareContext context, ProgramLocationService programLocationService, LocationService locationService, ExperimentUtilities experimentUtilities) { @@ -108,43 +108,43 @@ public static PendingLocation pendingLocation(ExpUnitMiddlewareContext context, @Bean @Prototype - public PendingTrial pendingTrialBean(ExpUnitMiddlewareContext context) { + public PendingTrial pendingTrialBean(AppendOverwriteMiddlewareContext context) { return pendingTrial(context, trialService, brapiTrialDAO, experimentUtilities); } @Bean @Prototype - public PendingObservationUnit pendingObservationUnitBean(ExpUnitMiddlewareContext context) { + public PendingObservationUnit pendingObservationUnitBean(AppendOverwriteMiddlewareContext context) { return pendingObservationUnit(context, observationUnitDAO, observationUnitService, experimentUtilities); } @Bean @Prototype - public PendingStudy pendingStudyBean(ExpUnitMiddlewareContext context) { + public PendingStudy pendingStudyBean(AppendOverwriteMiddlewareContext context) { return pendingStudy(context, studyService, brAPIStudyDAO, experimentUtilities); } @Bean @Prototype - public PendingGermplasm pendingGermplasmBean(ExpUnitMiddlewareContext context) { + public PendingGermplasm pendingGermplasmBean(AppendOverwriteMiddlewareContext context) { return pendingGermplasm(context, germplasmService, experimentUtilities); } @Bean @Prototype - public PendingDataset pendingDatasetBean(ExpUnitMiddlewareContext context) { + public PendingDataset pendingDatasetBean(AppendOverwriteMiddlewareContext context) { return pendingDataset(context, brAPIListDAO, datasetService, experimentUtilities); } @Bean @Prototype - public PendingObservation pendingObservationBean(ExpUnitMiddlewareContext context) { + public PendingObservation pendingObservationBean(AppendOverwriteMiddlewareContext context) { return pendingObservation(context, brAPIObservationDAO, ontologyService, experimentUtilities); } @Bean @Prototype - public PendingLocation pendingLocationBean(ExpUnitMiddlewareContext context) { + public PendingLocation pendingLocationBean(AppendOverwriteMiddlewareContext context) { return pendingLocation(context, programLocationService, locationService, experimentUtilities); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java index 2755a21ff..f8009d37b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java @@ -7,8 +7,8 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendWorkflowContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.GermplasmService; import org.breedinginsight.services.exceptions.DoesNotExistException; @@ -20,15 +20,15 @@ @Prototype public class PendingGermplasm implements ExperimentImportEntity { - AppendWorkflowContext cache; + AppendOverwriteWorkflowContext cache; ImportContext importContext; GermplasmService germplasmService; ExperimentUtilities experimentUtilities; - public PendingGermplasm(ExpUnitMiddlewareContext context, + public PendingGermplasm(AppendOverwriteMiddlewareContext context, GermplasmService germplasmService, ExperimentUtilities experimentUtilities) { - this.cache = context.getExpUnitContext(); + this.cache = context.getAppendOverwriteWorkflowContext(); this.importContext = context.getImportContext(); this.germplasmService = germplasmService; this.experimentUtilities = experimentUtilities; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java index 3dacbbc2f..537984844 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java @@ -8,8 +8,8 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendWorkflowContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; import org.breedinginsight.model.ProgramLocation; @@ -23,17 +23,17 @@ @Prototype public class PendingLocation implements ExperimentImportEntity { - AppendWorkflowContext cache; + AppendOverwriteWorkflowContext cache; ImportContext importContext; ProgramLocationService programLocationService; LocationService locationService; ExperimentUtilities experimentUtilities; - public PendingLocation(ExpUnitMiddlewareContext context, + public PendingLocation(AppendOverwriteMiddlewareContext context, ProgramLocationService programLocationService, LocationService locationService, ExperimentUtilities experimentUtilities) { - this.cache = context.getExpUnitContext(); + this.cache = context.getAppendOverwriteWorkflowContext(); this.importContext = context.getImportContext(); this.programLocationService = programLocationService; this.locationService = locationService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java index 76a3b7e12..e8ad7183b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java @@ -7,8 +7,8 @@ import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendWorkflowContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Trait; import org.breedinginsight.services.OntologyService; @@ -21,17 +21,17 @@ @Prototype public class PendingObservation implements ExperimentImportEntity { - AppendWorkflowContext cache; + AppendOverwriteWorkflowContext cache; ImportContext importContext; BrAPIObservationDAO brAPIObservationDAO; OntologyService ontologyService; ExperimentUtilities experimentUtilities; - public PendingObservation(ExpUnitMiddlewareContext context, + public PendingObservation(AppendOverwriteMiddlewareContext context, BrAPIObservationDAO brAPIObservationDAO, OntologyService ontologyService, ExperimentUtilities experimentUtilities) { - this.cache = context.getExpUnitContext(); + this.cache = context.getAppendOverwriteWorkflowContext(); this.importContext = context.getImportContext(); this.brAPIObservationDAO = brAPIObservationDAO; this.ontologyService = ontologyService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java index 240ece9cc..a829cc5df 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java @@ -9,8 +9,8 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendWorkflowContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; import org.breedinginsight.services.exceptions.DoesNotExistException; @@ -23,17 +23,17 @@ @Prototype public class PendingObservationUnit implements ExperimentImportEntity { - AppendWorkflowContext cache; + AppendOverwriteWorkflowContext cache; ImportContext importContext; BrAPIObservationUnitDAO observationUnitDAO; ObservationUnitService observationUnitService; ExperimentUtilities experimentUtilities; - public PendingObservationUnit(ExpUnitMiddlewareContext context, + public PendingObservationUnit(AppendOverwriteMiddlewareContext context, BrAPIObservationUnitDAO observationUnitDAO, ObservationUnitService observationUnitService, ExperimentUtilities experimentUtilities) { - this.cache = context.getExpUnitContext(); + this.cache = context.getAppendOverwriteWorkflowContext(); this.importContext = context.getImportContext(); this.observationUnitDAO = observationUnitDAO; this.observationUnitService = observationUnitService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java index 6b2e001d6..8774c3f7c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java @@ -7,8 +7,8 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendWorkflowContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; import org.breedinginsight.services.exceptions.DoesNotExistException; @@ -21,17 +21,17 @@ @Prototype public class PendingStudy implements ExperimentImportEntity{ - AppendWorkflowContext cache; + AppendOverwriteWorkflowContext cache; ImportContext importContext; StudyService studyService; BrAPIStudyDAO brAPIStudyDAO; ExperimentUtilities experimentUtilities; - public PendingStudy(ExpUnitMiddlewareContext context, + public PendingStudy(AppendOverwriteMiddlewareContext context, StudyService studyService, BrAPIStudyDAO brAPIStudyDAO, ExperimentUtilities experimentUtilities) { - this.cache = context.getExpUnitContext(); + this.cache = context.getAppendOverwriteWorkflowContext(); this.importContext = context.getImportContext(); this.studyService = studyService; this.brAPIStudyDAO = brAPIStudyDAO; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java index 27a339339..7a43bf333 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java @@ -7,8 +7,8 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendWorkflowContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; import org.breedinginsight.utilities.Utilities; @@ -18,17 +18,17 @@ @Prototype public class PendingTrial implements ExperimentImportEntity { - private final AppendWorkflowContext cache; + private final AppendOverwriteWorkflowContext cache; private final ImportContext importContext; private final TrialService trialService; private final BrAPITrialDAO brapiTrialDAO; private final ExperimentUtilities experimentUtilities; - public PendingTrial(ExpUnitMiddlewareContext context, + public PendingTrial(AppendOverwriteMiddlewareContext context, TrialService trialService, BrAPITrialDAO brapiTrialDAO, ExperimentUtilities experimentUtilities) { - this.cache = context.getExpUnitContext(); + this.cache = context.getAppendOverwriteWorkflowContext(); this.importContext = context.getImportContext(); this.trialService = trialService; this.brapiTrialDAO = brapiTrialDAO; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java similarity index 51% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java index 90879f23c..adda352a6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitIDValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java @@ -3,23 +3,23 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; @Slf4j @Prototype -public class ExpUnitIDValidation extends ExpUnitMiddleware { +public class AppendOverwriteIDValidation extends AppendOverwriteMiddleware { @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { - context.getExpUnitContext().setReferenceOUIds(ExperimentUtilities.collateReferenceOUIds(context)); + context.getAppendOverwriteWorkflowContext().setReferenceOUIds(ExperimentUtilities.collateReferenceOUIds(context)); return processNext(context); } @Override - public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { // tag an error if it occurred in this local transaction - context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); + context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); // undo the prior local transaction return compensatePrior(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java index c46a89a2d..b06558d33 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java @@ -10,8 +10,8 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; @@ -19,7 +19,7 @@ import java.util.stream.Collectors; @Slf4j -public class GetExistingBrAPIData extends ExpUnitMiddleware { +public class GetExistingBrAPIData extends AppendOverwriteMiddleware { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @@ -30,33 +30,33 @@ public GetExistingBrAPIData(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { } @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { return processNext(context); } @Override - public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { // tag an error if it occurred in this local transaction - context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); + context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); // handle the error in the prior local transaction return compensatePrior(context); } private Map> fetchReferenceObservationUnits( - ExpUnitMiddlewareContext context) { + AppendOverwriteMiddlewareContext context) { Map> pendingUnitById = new HashMap<>(); try { // Retrieve reference Observation Units based on IDs List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( - new ArrayList(context.getExpUnitContext().getReferenceOUIds()), + new ArrayList(context.getAppendOverwriteWorkflowContext().getReferenceOUIds()), context.getImportContext().getProgram() ); // Construct the DeltaBreed observation unit source for external references String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - if (referenceObsUnits.size() == context.getExpUnitContext().getReferenceOUIds().size()) { + if (referenceObsUnits.size() == context.getAppendOverwriteWorkflowContext().getReferenceOUIds().size()) { // Iterate through reference Observation Units for (BrAPIObservationUnit unit : referenceObsUnits) {// Get external reference for the Observation Unit @@ -82,7 +82,7 @@ private Map> fetchReferenceObs } } else { // Handle case of missing Observation Units in data store - List missingIds = new ArrayList<>(context.getExpUnitContext().getReferenceOUIds()); + List missingIds = new ArrayList<>(context.getAppendOverwriteWorkflowContext().getReferenceOUIds()); Set fetchedIds = referenceObsUnits.stream().map(unit -> Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) .orElseThrow(() -> new InternalServerException("External reference does not exist for Deltabreed ObservationUnit ID")) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java index 1bd68fb3d..002a26d36 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java @@ -2,20 +2,20 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; @Slf4j @Prototype -public class Transaction extends ExpUnitMiddleware { +public class Transaction extends AppendOverwriteMiddleware { @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { return processNext(context); } @Override - public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { return context; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java index c20d494cf..97ce72522 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java @@ -2,15 +2,15 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import javax.inject.Inject; @Slf4j @Prototype -public class BrAPICommit extends ExpUnitMiddleware { - ExpUnitMiddleware middleware; +public class BrAPICommit extends AppendOverwriteMiddleware { + AppendOverwriteMiddleware middleware; @Inject public BrAPICommit(BrAPIDatasetCommit brAPIDatasetCommit, BrAPITrialCommit brAPITrialCommit, @@ -22,7 +22,7 @@ public BrAPICommit(BrAPIDatasetCommit brAPIDatasetCommit, // TODO: add methods to entity/action classes to register and watch for required foreign key values so order does not have to be hard-wired // Note: the order is important because system-generated dbIds from prior steps are used as foreign keys in // subsequent steps - this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( + this.middleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link( brAPIDatasetCommit, brAPITrialCommit, locationCommit, @@ -32,7 +32,7 @@ public BrAPICommit(BrAPIDatasetCommit brAPIDatasetCommit, } @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { log.debug("starting post of experiment data to BrAPI server"); return this.middleware.process(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index 7d0c6ba96..58f418781 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -8,8 +8,8 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import javax.inject.Inject; @@ -17,7 +17,7 @@ @Slf4j @Prototype -public class BrAPIDatasetCommit extends ExpUnitMiddleware { +public class BrAPIDatasetCommit extends AppendOverwriteMiddleware { private final BrAPICreationFactory brAPICreationFactory; private final BrAPIUpdateFactory brAPIUpdateFactory; private WorkflowCreation datasetCreation; @@ -32,7 +32,7 @@ public BrAPIDatasetCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUpdate this.brAPIUpdateFactory = brAPIUpdateFactory; } @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { try { datasetCreation = brAPICreationFactory.datasetWorkflowCreationBean(context); @@ -43,16 +43,16 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { log.info("adding new observation variables to datasets"); updatedDatasets = datasetUpdate.execute().map(d -> (WorkflowUpdate.BrAPIUpdateState) d); } catch (ApiException e) { - context.getExpUnitContext().setProcessError(new MiddlewareError(e)); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareError(e)); return this.compensate(context); } return processNext(context); } @Override - public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { // Tag an error if it occurred in this local transaction - context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); + context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); // Delete any created datasets createdDatasets.ifPresent(WorkflowCreation.BrAPICreationState::undo); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java index 9881d13e1..b940f3989 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -8,8 +8,8 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import javax.inject.Inject; @@ -17,7 +17,7 @@ @Slf4j @Prototype -public class BrAPIObservationCommit extends ExpUnitMiddleware { +public class BrAPIObservationCommit extends AppendOverwriteMiddleware { private final BrAPICreationFactory brAPICreationFactory; private final BrAPIUpdateFactory brAPIUpdateFactory; private WorkflowCreation brAPIObservationCreation; @@ -32,7 +32,7 @@ public BrAPIObservationCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUp this.brAPIUpdateFactory = brAPIUpdateFactory; } @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { try { brAPIObservationCreation = brAPICreationFactory.observationWorkflowCreationBean(context); log.info("creating new observations in the BrAPI service"); @@ -43,7 +43,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { updatedObservations = brAPIObservationUpdate.execute().map(s -> (WorkflowUpdate.BrAPIUpdateState) s); } catch (ApiException e) { - context.getExpUnitContext().setProcessError(new MiddlewareError(e)); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareError(e)); return this.compensate(context); } @@ -51,9 +51,9 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { } @Override - public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { // Tag an error if it occurred in this local transaction - context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); + context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); // Delete any created observations from the BrAPI service createdBrAPIObservations.ifPresent(WorkflowCreation.BrAPICreationState::undo); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index c2f74dab3..3a766a877 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -6,8 +6,8 @@ import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import javax.inject.Inject; @@ -15,7 +15,7 @@ @Slf4j @Prototype -public class BrAPIObservationUnitCommit extends ExpUnitMiddleware { +public class BrAPIObservationUnitCommit extends AppendOverwriteMiddleware { private BrAPICreationFactory brAPICreationFactory; private WorkflowCreation brAPIObservationUnitCreation; private Optional createdBrAPIObservationUnits; @@ -25,22 +25,22 @@ public BrAPIObservationUnitCommit(BrAPICreationFactory brAPICreationFactory) { this.brAPICreationFactory = brAPICreationFactory; } @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { try{ brAPIObservationUnitCreation = brAPICreationFactory.observationUnitWorkflowCreationBean(context); log.info("creating new observation units in the BrAPI service"); createdBrAPIObservationUnits = brAPIObservationUnitCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); } catch (ApiException e) { - context.getExpUnitContext().setProcessError(new MiddlewareError(e)); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareError(e)); return this.compensate(context); } return processNext(context); } @Override - public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { // Tag an error if it occurred in this local transaction - context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); + context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); // Delete any created trials from the BrAPI service createdBrAPIObservationUnits.ifPresent(WorkflowCreation.BrAPICreationState::undo); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java index 8cc62023e..24a1039d9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -6,8 +6,8 @@ import org.brapi.v2.model.core.BrAPIStudy; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import javax.inject.Inject; @@ -15,7 +15,7 @@ @Slf4j @Prototype -public class BrAPIStudyCommit extends ExpUnitMiddleware { +public class BrAPIStudyCommit extends AppendOverwriteMiddleware { private BrAPICreationFactory brAPICreationFactory; private WorkflowCreation brAPIStudyCreation; private Optional createdBrAPIStudies; @@ -25,13 +25,13 @@ public BrAPIStudyCommit(BrAPICreationFactory brAPICreationFactory) { this.brAPICreationFactory = brAPICreationFactory; } @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { try { brAPIStudyCreation = brAPICreationFactory.studyWorkflowCreationBean(context); log.info("creating new studies in the BrAPI service"); createdBrAPIStudies = brAPIStudyCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); } catch (ApiException e) { - context.getExpUnitContext().setProcessError(new MiddlewareError(e)); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareError(e)); return this.compensate(context); } @@ -39,9 +39,9 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { } @Override - public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { // Tag an error if it occurred in this local transaction - context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); + context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); // Delete any created studies from the BrAPI service createdBrAPIStudies.ifPresent(WorkflowCreation.BrAPICreationState::undo); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index a4dcd5a4b..d2f6337a2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -8,8 +8,8 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import javax.inject.Inject; @@ -17,7 +17,7 @@ @Slf4j @Prototype -public class BrAPITrialCommit extends ExpUnitMiddleware { +public class BrAPITrialCommit extends AppendOverwriteMiddleware { private final BrAPICreationFactory brAPICreationFactory; private final BrAPIUpdateFactory brAPIUpdateFactory; private WorkflowCreation brAPITrialCreation; @@ -32,7 +32,7 @@ public BrAPITrialCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUpdateFa this.brAPIUpdateFactory = brAPIUpdateFactory; } @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { try { brAPITrialCreation = brAPICreationFactory.trialWorkflowCreationBean(context); log.info("creating new trials in the BrAPI service"); @@ -43,7 +43,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { updatedTrials = brAPITrialUpdate.execute().map(s -> (WorkflowUpdate.BrAPIUpdateState) s); } catch (ApiException e) { - context.getExpUnitContext().setProcessError(new MiddlewareError(e)); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareError(e)); return this.compensate(context); } @@ -51,9 +51,9 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { } @Override - public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { // Tag an error if it occurred in this local transaction - context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); + context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); // Delete any created trials from the BrAPI service createdBrAPITrials.ifPresent(WorkflowCreation.BrAPICreationState::undo); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index 4b05c7246..13033d104 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -5,8 +5,8 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import org.breedinginsight.model.ProgramLocation; @@ -15,7 +15,7 @@ @Slf4j @Prototype -public class LocationCommit extends ExpUnitMiddleware { +public class LocationCommit extends AppendOverwriteMiddleware { private BrAPICreationFactory brAPICreationFactory; private WorkflowCreation locationCreation; private Optional createdLocations; @@ -25,22 +25,22 @@ public LocationCommit(BrAPICreationFactory brAPICreationFactory) { this.brAPICreationFactory = brAPICreationFactory; } @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { try { locationCreation = brAPICreationFactory.locationWorkflowCreationBean(context); log.info("creating new locationss in the Deltabreed database"); createdLocations = locationCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); } catch (ApiException e) { - context.getExpUnitContext().setProcessError(new MiddlewareError(e)); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareError(e)); return this.compensate(context); } return processNext(context); } @Override - public ExpUnitMiddlewareContext compensate(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { // Tag an error if it occurred in this local transaction - context.getExpUnitContext().getProcessError().tag(this.getClass().getName()); + context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); // Delete any created locations createdLocations.ifPresent(WorkflowCreation.BrAPICreationState::undo); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index e9b141121..6bf87837a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -10,8 +10,8 @@ import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIReadFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowReadInitialization; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import org.breedinginsight.model.ProgramLocation; @@ -19,7 +19,7 @@ @Slf4j @Prototype -public class WorkflowInitialization extends ExpUnitMiddleware { +public class WorkflowInitialization extends AppendOverwriteMiddleware { WorkflowReadInitialization brAPIObservationUnitReadWorkflowInitialization; WorkflowReadInitialization brAPITrialReadWorkflowInitialization; WorkflowReadInitialization brAPIStudyReadWorkflowInitialization; @@ -33,7 +33,7 @@ public WorkflowInitialization(BrAPIReadFactory brAPIReadFactory) { this.brAPIReadFactory = brAPIReadFactory; } @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { brAPIObservationUnitReadWorkflowInitialization = brAPIReadFactory.observationUnitWorkflowReadInitializationBean(context); brAPITrialReadWorkflowInitialization = brAPIReadFactory.trialWorkflowReadInitializationBean(context); brAPIStudyReadWorkflowInitialization = brAPIReadFactory.studyWorkflowReadInitializationBean(context); @@ -50,7 +50,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { brAPIDatasetReadWorkflowInitialization.execute(); brAPIGermplasmReadWorkflowInitialization.execute(); } catch (ApiException e) { - context.getExpUnitContext().setProcessError(new MiddlewareError(e)); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareError(e)); return this.compensate(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 8fe06dca1..2b9181fd5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -25,9 +25,9 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.ProcessedDataFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.VisitedObservationData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationVariableService; @@ -52,7 +52,7 @@ @Slf4j @Prototype -public class ImportTableProcess extends ExpUnitMiddleware { +public class ImportTableProcess extends AppendOverwriteMiddleware { @Property(name = "brapi.server.reference-source") String brapiReferenceSource; StudyService studyService; @@ -86,7 +86,7 @@ public ImportTableProcess(StudyService studyService, } @Override - public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { log.debug("verifying traits listed in import"); // Get all the dynamic columns of the import @@ -105,11 +105,11 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Construct validation errors for any timestamp columns that don't have a matching variable column List importRows = context.getImportContext().getImportRows(); - Optional.ofNullable(context.getExpUnitContext().getValidationErrors()).orElseGet(() -> { - context.getExpUnitContext().setValidationErrors(new ValidationErrors()); + Optional.ofNullable(context.getAppendOverwriteWorkflowContext().getValidationErrors()).orElseGet(() -> { + context.getAppendOverwriteWorkflowContext().setValidationErrors(new ValidationErrors()); return new ValidationErrors(); }); - ValidationErrors validationErrors = context.getExpUnitContext().getValidationErrors(); + ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); List tsValErrs = observationVariableService.validateMatchedTimestamps(Set.copyOf(varNames), timestampCols).orElse(new ArrayList<>()); for (int i = 0; i < importRows.size(); i++) { int rowNum = i; @@ -126,7 +126,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { Map> tsColByPheno = timestampCols.stream().collect(Collectors.toMap(col -> col.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), col -> col)); // Add the map to the context for use in processing import - context.getExpUnitContext().setTimeStampColByPheno(tsColByPheno); + context.getAppendOverwriteWorkflowContext().setTimeStampColByPheno(tsColByPheno); // Fetch the traits named in the observation variable columns Program program = context.getImportContext().getProgram(); @@ -146,9 +146,9 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { List sortedTraits = experimentUtil.sortByField(varNames, new ArrayList<>(traits), TraitEntity::getObservationVariableName); // Get the pending observation dataset - PendingImportObject pendingTrial = ExperimentUtilities.getSingleEntryValue(context.getExpUnitContext().getTrialByNameNoScope()).orElseThrow(()->new UnprocessableEntityException(MULTIPLE_EXP_TITLES.getValue())); + PendingImportObject pendingTrial = ExperimentUtilities.getSingleEntryValue(context.getAppendOverwriteWorkflowContext().getTrialByNameNoScope()).orElseThrow(()->new UnprocessableEntityException(MULTIPLE_EXP_TITLES.getValue())); String datasetName = String.format("Observation Dataset [%s-%s]", program.getKey(), pendingTrial.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString()); - PendingImportObject pendingDataset = context.getExpUnitContext().getObsVarDatasetByName().get(datasetName); + PendingImportObject pendingDataset = context.getAppendOverwriteWorkflowContext().getObsVarDatasetByName().get(datasetName); // Add new phenotypes to the pending observation dataset list (NOTE: "obsVarName [programKey]" is used instead of obsVarDbId) // TODO: Change to using actual dbIds as per the BrAPI spec, instead of namespaced obsVar names (was necessary for Breedbase) @@ -164,14 +164,14 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // Read any observation data stored for these traits log.debug("fetching observation data stored for traits"); - Set ouDbIds = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(u -> u.getBrAPIObject().getObservationUnitDbId()).collect(Collectors.toSet()); + Set ouDbIds = context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().values().stream().map(u -> u.getBrAPIObject().getObservationUnitDbId()).collect(Collectors.toSet()); Set varDbIds = sortedTraits.stream().map(t->t.getObservationVariableDbId()).collect(Collectors.toSet()); List observations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, varDbIds, program); // Construct helper lookup tables to use for hashing stored observation data - Map unitNameByDbId = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, BrAPIObservationUnit::getObservationUnitName)); + Map unitNameByDbId = context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, BrAPIObservationUnit::getObservationUnitName)); Map variableNameByDbId = sortedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableDbId, Trait::getObservationVariableName)); - Map studyNameByDbId = context.getExpUnitContext().getStudyByNameNoScope().values().stream() + Map studyNameByDbId = context.getAppendOverwriteWorkflowContext().getStudyByNameNoScope().values().stream() .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) .map(PendingImportObject::getBrAPIObject) .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); @@ -184,7 +184,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { }, o->o)); // Add the observation data map to the context for use in processing import - context.getExpUnitContext().setExistingObsByObsHash(observationByObsHash); + context.getAppendOverwriteWorkflowContext().setExistingObsByObsHash(observationByObsHash); // Build new pending observation data for each phenotype Map> pendingObservationByHash = new HashMap<>(); @@ -201,19 +201,19 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { }); PendingImport mappedImportRow = context.getImportContext().getMappedBrAPIImport().getOrDefault(rowNum, new PendingImport()); String unitId = row.getObsUnitID(); - mappedImportRow.setTrial(context.getExpUnitContext().getPendingTrialByOUId().get(unitId)); - mappedImportRow.setLocation(context.getExpUnitContext().getPendingLocationByOUId().get(unitId)); - mappedImportRow.setStudy(context.getExpUnitContext().getPendingStudyByOUId().get(unitId)); - mappedImportRow.setObservationUnit(context.getExpUnitContext().getPendingObsUnitByOUId().get(unitId)); - mappedImportRow.setGermplasm(context.getExpUnitContext().getPendingGermplasmByOUId().get(unitId)); + mappedImportRow.setTrial(context.getAppendOverwriteWorkflowContext().getPendingTrialByOUId().get(unitId)); + mappedImportRow.setLocation(context.getAppendOverwriteWorkflowContext().getPendingLocationByOUId().get(unitId)); + mappedImportRow.setStudy(context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId)); + mappedImportRow.setObservationUnit(context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(unitId)); + mappedImportRow.setGermplasm(context.getAppendOverwriteWorkflowContext().getPendingGermplasmByOUId().get(unitId)); // Assemble the pending observation data for all phenotypes for (Column column : phenotypeCols) { String cellData = column.getString(rowNum); // Generate hash for looking up prior observation data - String studyName = context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getStudyName(); - String unitName = context.getExpUnitContext().getPendingObsUnitByOUId().get(unitId).getBrAPIObject().getObservationUnitName(); + String studyName = context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getStudyName(); + String unitName = context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(unitId).getBrAPIObject().getObservationUnitName(); String phenoColumnName = column.name(); String observationHash = observationService.getObservationHash(unitName, phenoColumnName, studyName); @@ -272,22 +272,22 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { } else { // Clone the observation unit and trait - BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getExpUnitContext().getPendingObsUnitByOUId().get(row.getObsUnitID()).getBrAPIObject()), BrAPIObservationUnit.class); + BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(row.getObsUnitID()).getBrAPIObject()), BrAPIObservationUnit.class); Trait initialTrait = gson.fromJson(gson.toJson(traitByPhenoColName.get(phenoColumnName)), Trait.class); // create new instance of InitialData processedData = processedDataFactory.initialDataBean(brapiReferenceSource, context.getImportContext().isCommit(), - context.getExpUnitContext().getPendingGermplasmByOUId().get(unitId).getBrAPIObject().getGermplasmName(), - context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject(), + context.getAppendOverwriteWorkflowContext().getPendingGermplasmByOUId().get(unitId).getBrAPIObject().getGermplasmName(), + context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getBrAPIObject(), cellData, phenoColumnName, initialTrait, row, pendingTrial.getId(), - context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getId(), + context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getId(), UUID.fromString(unitId), - context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getSeasons().get(0), + context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getSeasons().get(0), observationUnit, context.getImportContext().getUser(), context.getImportContext().getProgram()); @@ -306,7 +306,7 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { // TODO: change signature to take two args, studyName and unitName statistic.addObservationUnitId(null); statistic.addGid(null); - context.getExpUnitContext().setStatistic(statistic); + context.getAppendOverwriteWorkflowContext().setStatistic(statistic); // Construct a pending observation PendingImportObject pendingProcessedData = processedData.constructPendingObservation(); @@ -323,11 +323,11 @@ public ExpUnitMiddlewareContext process(ExpUnitMiddlewareContext context) { } // Add the pending observation map to the context for use in processing the import - context.getExpUnitContext().setPendingObservationByHash(pendingObservationByHash); + context.getAppendOverwriteWorkflowContext().setPendingObservationByHash(pendingObservationByHash); return processNext(context); } catch (DoesNotExistException | ApiException | UnprocessableEntityException | ValidatorException e) { - context.getExpUnitContext().setProcessError(new MiddlewareError(e)); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareError(e)); return this.compensate(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteMiddleware.java similarity index 71% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitMiddleware.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteMiddleware.java index 9fb5c04da..799b7847e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteMiddleware.java @@ -3,7 +3,7 @@ /** * ExpUnitMiddleware class extends Middleware class to handle compensating transactions in the context of ExpUnitMiddlewareContext. */ -public abstract class ExpUnitMiddleware extends Middleware { +public abstract class AppendOverwriteMiddleware extends Middleware { /** * Compensates for an error that occurred in the current local transaction, tagging the error and undoing the previous local transaction. @@ -12,9 +12,9 @@ public abstract class ExpUnitMiddleware extends Middleware referenceOUIds = new HashSet<>(); private Map> pendingTrialByOUId = new HashMap<>(); From a08a82d538d71fcf006b21038db5c9b2de2bb286 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Sun, 23 Jun 2024 15:12:18 -0400 Subject: [PATCH 159/203] delete unnecessary transaction and BrAPIReadRequiredData middleware --- .../AppendOverwritePhenotypesWorkflow.java | 11 +- .../middleware/GetExistingBrAPIData.java | 105 ------------------ .../middleware/Transaction.java | 22 ---- 3 files changed, 4 insertions(+), 134 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 308821f31..26465ed0e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -5,18 +5,17 @@ import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; import org.breedinginsight.brapps.importer.services.ImportStatusService; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.AppendOverwriteIDValidation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.Transaction; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.BrAPICommit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize.WorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.ImportTableProcess; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; @@ -34,8 +33,7 @@ public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow { private final ImportStatusService statusService; @Inject - public AppendOverwritePhenotypesWorkflow(Transaction transaction, - AppendOverwriteIDValidation expUnitIDValidation, + public AppendOverwritePhenotypesWorkflow(AppendOverwriteIDValidation expUnitIDValidation, WorkflowInitialization workflowInitialization, ImportTableProcess importTableProcess, BrAPICommit brAPICommit, @@ -43,7 +41,6 @@ public AppendOverwritePhenotypesWorkflow(Transaction transaction, this.statusService = statusService; this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE; this.importPreviewMiddleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link( - transaction, expUnitIDValidation, workflowInitialization, importTableProcess); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java deleted file mode 100644 index b06558d33..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java +++ /dev/null @@ -1,105 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; - -import io.micronaut.context.annotation.Property; -import io.micronaut.http.server.exceptions.InternalServerException; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; -import org.breedinginsight.utilities.Utilities; - -import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; - -@Slf4j -public class GetExistingBrAPIData extends AppendOverwriteMiddleware { - private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; - @Property(name = "brapi.server.reference-source") - private String BRAPI_REFERENCE_SOURCE; - - @Inject - public GetExistingBrAPIData(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { - this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; - } - - @Override - public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { - - return processNext(context); - } - - @Override - public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { - // tag an error if it occurred in this local transaction - context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); - - // handle the error in the prior local transaction - return compensatePrior(context); - } - private Map> fetchReferenceObservationUnits( - AppendOverwriteMiddlewareContext context) { - Map> pendingUnitById = new HashMap<>(); - try { - // Retrieve reference Observation Units based on IDs - List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( - new ArrayList(context.getAppendOverwriteWorkflowContext().getReferenceOUIds()), - context.getImportContext().getProgram() - ); - - // Construct the DeltaBreed observation unit source for external references - String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - - if (referenceObsUnits.size() == context.getAppendOverwriteWorkflowContext().getReferenceOUIds().size()) { - - // Iterate through reference Observation Units - for (BrAPIObservationUnit unit : referenceObsUnits) {// Get external reference for the Observation Unit - Optional unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource); - unitXref.ifPresentOrElse( - xref -> { - - // Set pending Observation Unit by its ID - pendingUnitById.put( - xref.getReferenceId(), - new PendingImportObject<>( - ImportObjectState.EXISTING, unit, UUID.fromString(xref.getReferenceId())) - ); - }, - () -> { - - // but throw an error if no unit ID - this.compensate(context); - } - ); - - - } - } else { - // Handle case of missing Observation Units in data store - List missingIds = new ArrayList<>(context.getAppendOverwriteWorkflowContext().getReferenceOUIds()); - Set fetchedIds = referenceObsUnits.stream().map(unit -> - Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) - .orElseThrow(() -> new InternalServerException("External reference does not exist for Deltabreed ObservationUnit ID")) - .getReferenceId()) - .collect(Collectors.toSet()); - missingIds.removeAll(fetchedIds); - - // throw error reporting any reference IDs with no corresponding stored unit in the brapi data store - this.compensate(context); - } - - return pendingUnitById; - } catch (ApiException e) { - - // throw an error if problem getting data from the brapi data store - this.compensate(context); - } - return pendingUnitById; - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java deleted file mode 100644 index 002a26d36..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; - -import io.micronaut.context.annotation.Prototype; -import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; - -@Slf4j -@Prototype -public class Transaction extends AppendOverwriteMiddleware { - - @Override - public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { - return processNext(context); - } - - @Override - public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { - - return context; - } -} From b7ece342b0145e7ada6834eef0a506ebb0046e66 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Sun, 23 Jun 2024 22:46:13 -0400 Subject: [PATCH 160/203] move throwing exceptions to experiment import service layer --- .../model/imports/DomainImportService.java | 16 ++-- .../model/workflow/ImportWorkflowResult.java | 1 + .../ExperimentWorkflowNavigator.java | 18 +++- .../AppendOverwritePhenotypesWorkflow.java | 24 +++-- .../middleware/commit/BrAPIDatasetCommit.java | 4 +- .../commit/BrAPIObservationCommit.java | 4 +- .../commit/BrAPIObservationUnitCommit.java | 4 +- .../middleware/commit/BrAPIStudyCommit.java | 4 +- .../middleware/commit/BrAPITrialCommit.java | 4 +- .../middleware/commit/LocationCommit.java | 4 +- .../initialize/WorkflowInitialization.java | 4 +- .../process/ImportTableProcess.java | 4 +- .../model/AppendOverwriteWorkflowContext.java | 2 +- ...areError.java => MiddlewareException.java} | 8 +- .../workflow/CreateNewExperimentWorkflow.java | 55 ------------ .../workflow/NewExperimentWorkflow.java | 88 +++++++++++++++++++ 16 files changed, 153 insertions(+), 91 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/{MiddlewareError.java => MiddlewareException.java} (72%) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 071c9f0ef..15c839799 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -24,8 +24,8 @@ import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; -import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; import java.util.List; @@ -67,11 +67,17 @@ public ImportPreviewResponse process(ImportServiceContext context) .filter(workflow -> !workflow.isEmpty()) .ifPresent(workflow -> log.info("Workflow: " + workflow)); - // TODO: return results from WorkflowNavigator once processing logic is in separate workflows - // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); - if ("append-dataset".equals(context.getWorkflow())) { - return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); + if (ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE.getId().equals(context.getWorkflow()) || ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION.getId().equals(context.getWorkflow())) { + Optional result = workflowNavigator.process(context); + + // Throw any exceptions caught during workflow processing + result.flatMap(ImportWorkflowResult::getCaughtException).ifPresent(error -> { + throw new RuntimeException(error.getCause()); + }); + + return result.flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); } else { + // TODO: remove this case and just use workflow navigator once all integration tests have been updated to use workflow ids return processorManagerProvider.get().process(context.getBrAPIImports(), List.of(experimentProcessorProvider.get()), context.getData(), diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java index 9304c1ce2..f59b83b55 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java @@ -30,4 +30,5 @@ public class ImportWorkflowResult { private ImportWorkflow workflow; private Optional importPreviewResponse; + private Optional caughtException; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index 63bfcedbd..b1ed662bb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -16,12 +16,20 @@ public class ExperimentWorkflowNavigator implements ExperimentWorkflow { private final List workflows; + /** Micronaut scans and collects in a List all instances of ExperimentWorkflow not annotated as @Primary and + * automatically makes the list available to inject into the constructor, which is set here to the workflows field. + * The order in the list is determined by the sort value returned from each instance by calling getOrder(). + * Instances returning a lower sort value will appear in the list before instances returning higher sort values. + */ public ExperimentWorkflowNavigator(List workflows) { this.workflows = workflows; } @Override public Optional process(ImportServiceContext context) { + /** + * Have each workflow in order process the context, returning the first non-empty result + */ return workflows.stream() .map(workflow->workflow.process(context)) .filter(Optional::isPresent) @@ -29,8 +37,9 @@ public Optional process(ImportServiceContext context) { .findFirst(); } public List getWorkflows() { - // Each workflow returns in the field workflow the metadata about the workflow that processed the import context. - // Loop over all workflows, processing a null context, to collect just the metadata + /** Each workflow returns in the field workflow the metadata about the workflow that processed the import context. + * Loop over all workflows, processing a null context, to collect just the metadata for each workflow + */ List workflowSummaryList = workflows.stream() .map(workflow->workflow.process(null)) .filter(Optional::isPresent) @@ -38,7 +47,7 @@ public List getWorkflows() { .map(result->result.getWorkflow()) .collect(Collectors.toList()); - // The order field for each workflow is set to the order in the list + // The order field for each workflow is set to the order in the navigator list for (int i = 0; i < workflowSummaryList.size(); i++) { workflowSummaryList.get(i).setOrder(i); } @@ -46,6 +55,9 @@ public List getWorkflows() { return workflowSummaryList; } + /** + * Possible choices of metadata that an experiment workflow instance can use to identify itself + */ public enum Workflow { NEW_OBSERVATION("new-experiment","Create new experiment"), APPEND_OVERWRITE("append-dataset", "Append experimental dataset"), diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 26465ed0e..c0dcaeb8b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -17,6 +17,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import javax.inject.Inject; @@ -59,6 +60,7 @@ public Optional process(ImportServiceContext context) { Optional result = Optional.of(ImportWorkflowResult.builder() .workflow(workflow) // attach metadata of this workflow to response .importPreviewResponse(Optional.empty()) + .caughtException(Optional.empty()) .build()); // Skip this workflow unless appending or overwriting observation data @@ -71,7 +73,7 @@ public Optional process(ImportServiceContext context) { return result; } - // Build the context for processing the import + // Build the workflow context for processing the import ImportContext importContext = ImportContext.builder() .upload(context.getUpload()) .importRows(context.getBrAPIImports()) @@ -88,12 +90,13 @@ public Optional process(ImportServiceContext context) { // Process the import preview AppendOverwriteMiddlewareContext processedPreviewContext = this.importPreviewMiddleware.process(workflowContext); - // TODO: Rethrow any exceptions caught during processing the context -// Optional.ofNullable(processedContext.getExpUnitContext().getMiddlewareError()).ifPresent(e -> { -// Constructor constructor = e.getError().getClass().getConstructor(String.class, Throwable.class); -// Exception newException = constructor.newInstance(e.getError().getMessage(), e); -// throw newException; -// }); + // Stop and return any errors that occurred while processing + Optional previewException = Optional.ofNullable(processedPreviewContext.getAppendOverwriteWorkflowContext().getProcessError()); + if (previewException.isPresent() ) { + log.debug(String.format("%s in %s", previewException.get().getException().getClass()), previewException.get().getLocalTransactionName()); + result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.ofNullable(previewException.get().getException()))); + return result; + } // BUild and return the preview response ImportPreviewResponse response = new ImportPreviewResponse(); @@ -123,6 +126,13 @@ public Optional process(ImportServiceContext context) { // Commit the changes from the processed import preview to the BrAPI service AppendOverwriteMiddlewareContext brapiCommittedContext = this.brapiCommitMiddleware.process(processedPreviewContext); + Optional brapiCommitException = Optional.ofNullable(brapiCommittedContext.getAppendOverwriteWorkflowContext().getProcessError()); + if (brapiCommitException.isPresent() ) { + log.debug(String.format("%s in %s", brapiCommitException.get().getException().getClass()), brapiCommitException.get().getLocalTransactionName()); + result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.ofNullable(brapiCommitException.get().getException()))); + return result; + } + log.debug("Completed upload to brapi service"); statusService.finishUpload(context.getUpload(), totalObjects, "Completed upload to brapi service"); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index 58f418781..a7ef0b53b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -10,7 +10,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; import javax.inject.Inject; import java.util.Optional; @@ -43,7 +43,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext log.info("adding new observation variables to datasets"); updatedDatasets = datasetUpdate.execute().map(d -> (WorkflowUpdate.BrAPIUpdateState) d); } catch (ApiException e) { - context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareError(e)); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); } return processNext(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java index b940f3989..458369403 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -10,7 +10,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; import javax.inject.Inject; import java.util.Optional; @@ -43,7 +43,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext updatedObservations = brAPIObservationUpdate.execute().map(s -> (WorkflowUpdate.BrAPIUpdateState) s); } catch (ApiException e) { - context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareError(e)); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index 3a766a877..baa6eec48 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -8,7 +8,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; import javax.inject.Inject; import java.util.Optional; @@ -31,7 +31,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext log.info("creating new observation units in the BrAPI service"); createdBrAPIObservationUnits = brAPIObservationUnitCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); } catch (ApiException e) { - context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareError(e)); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); } return processNext(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java index 24a1039d9..b2ce28374 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -8,7 +8,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; import javax.inject.Inject; import java.util.Optional; @@ -31,7 +31,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext log.info("creating new studies in the BrAPI service"); createdBrAPIStudies = brAPIStudyCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); } catch (ApiException e) { - context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareError(e)); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index d2f6337a2..1b62389e4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -10,7 +10,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; import javax.inject.Inject; import java.util.Optional; @@ -43,7 +43,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext updatedTrials = brAPITrialUpdate.execute().map(s -> (WorkflowUpdate.BrAPIUpdateState) s); } catch (ApiException e) { - context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareError(e)); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index 13033d104..0cb3e0f4a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -7,7 +7,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; import org.breedinginsight.model.ProgramLocation; import javax.inject.Inject; @@ -31,7 +31,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext log.info("creating new locationss in the Deltabreed database"); createdLocations = locationCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); } catch (ApiException e) { - context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareError(e)); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); } return processNext(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index 6bf87837a..3da2b847f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -12,7 +12,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowReadInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; import org.breedinginsight.model.ProgramLocation; import javax.inject.Inject; @@ -50,7 +50,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext brAPIDatasetReadWorkflowInitialization.execute(); brAPIGermplasmReadWorkflowInitialization.execute(); } catch (ApiException e) { - context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareError(e)); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 2b9181fd5..3882785e6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -28,7 +28,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationVariableService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; @@ -327,7 +327,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext return processNext(context); } catch (DoesNotExistException | ApiException | UnprocessableEntityException | ValidatorException e) { - context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareError(e)); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java index 164552d46..2d4098612 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java @@ -35,7 +35,7 @@ public class AppendOverwriteWorkflowContext { private AppendStatistic statistic; // Exceptions - private MiddlewareError processError; + private MiddlewareException processError; private ValidationErrors validationErrors; // Cache maps keyed by name without program scope diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/MiddlewareError.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/MiddlewareException.java similarity index 72% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/MiddlewareError.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/MiddlewareException.java index 20850e5f0..3b8b79c85 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/MiddlewareError.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/MiddlewareException.java @@ -3,17 +3,17 @@ import lombok.Getter; import lombok.Setter; -public class MiddlewareError { +public class MiddlewareException { @Getter @Setter String localTransactionName; @Getter @Setter - Exception error; + Exception exception; - public MiddlewareError(Exception error) { - this.error = error; + public MiddlewareException(Exception exception) { + this.exception = exception; } public void tag(String name) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java deleted file mode 100644 index 503e4e5f0..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; - -import lombok.Getter; -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; - -import javax.inject.Singleton; -import java.util.Optional; - -@Getter -@Singleton -public class CreateNewExperimentWorkflow implements ExperimentWorkflow { - private final ExperimentWorkflowNavigator.Workflow workflow; - - public CreateNewExperimentWorkflow(){ - this.workflow = ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION; - } - - @Override - public Optional process(ImportServiceContext context) { - // Workflow processing the context - ImportWorkflow workflow = ImportWorkflow.builder() - .id(getWorkflow().getId()) - .name(getWorkflow().getName()) - .build(); - - // No-preview result - Optional result = Optional.of(ImportWorkflowResult.builder() - .workflow(workflow) - .importPreviewResponse(Optional.empty()) - .build()); - - // Skip this workflow unless creating a new experiment - if (context != null && !this.workflow.isEqual(context.getWorkflow())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return result; - } - - // Start processing the import... - return result; - } - - @Override - public int getOrder() { - return 1; - } - -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java new file mode 100644 index 000000000..0b67766a5 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java @@ -0,0 +1,88 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.services.ImportStatusService; +import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; +import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.List; +import java.util.Optional; + +@Slf4j +@Getter +@Singleton +public class NewExperimentWorkflow implements ExperimentWorkflow { + private final ExperimentWorkflowNavigator.Workflow workflow; + private final ImportStatusService statusService; + private final Provider experimentProcessorProvider; + private final Provider processorManagerProvider; + + @Inject + public NewExperimentWorkflow(ImportStatusService statusService, + Provider experimentProcessorProvider, + Provider processorManagerProvider){ + this.statusService = statusService; + this.experimentProcessorProvider = experimentProcessorProvider; + this.processorManagerProvider = processorManagerProvider; + this.workflow = ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION; + } + + @Override + public Optional process(ImportServiceContext context) { + // Metadata about this workflow processing the context + ImportWorkflow workflow = ImportWorkflow.builder() + .id(getWorkflow().getId()) + .name(getWorkflow().getName()) + .build(); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .workflow(workflow) // attach metadata of this workflow to response + .importPreviewResponse(Optional.empty()) + .caughtException(Optional.empty()) + .build()); + + // Skip this workflow unless creating a new experiment + if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result with metadata for this workflow + if (context == null) { + return result; + } + + // Build and return the preview response + try { + ImportPreviewResponse successResponse; + successResponse = processorManagerProvider.get().process(context.getBrAPIImports(), + List.of(experimentProcessorProvider.get()), + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); + result.ifPresent(importWorkflowResult -> importWorkflowResult.setImportPreviewResponse(Optional.of(successResponse))); + } catch (Exception e) { + result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.of(e))); + } + + return result; + } + + @Override + public int getOrder() { + return 1; + } + +} From 7733d1b1a496a6e916132a68a09f8403755b52cc Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Sun, 23 Jun 2024 23:17:22 -0400 Subject: [PATCH 161/203] add license statements --- .../model/workflow/ExperimentWorkflow.java | 17 + .../model/workflow/GermplasmWorkflow.java | 17 + .../workflow/SampleSubmissionWorkflow.java | 17 + .../importer/model/workflow/Workflow.java | 17 + .../experiment/ExperimentUtilities.java | 17 + .../ExperimentWorkflowNavigator.java | 17 + .../AppendOverwritePhenotypesWorkflow.java | 17 + .../appendoverwrite/factory/BrAPIState.java | 17 + .../factory/action/BrAPIAction.java | 17 + .../factory/action/BrAPICreationFactory.java | 17 + .../factory/action/BrAPIReadFactory.java | 17 + .../factory/action/BrAPIUpdateFactory.java | 17 + .../factory/action/WorkflowCreation.java | 17 + .../action/WorkflowReadInitialization.java | 17 + .../factory/action/WorkflowUpdate.java | 17 + .../factory/data/InitialData.java | 17 + .../factory/data/OverwrittenData.java | 17 + .../factory/data/ProcessedDataFactory.java | 17 + .../factory/data/UnchangedData.java | 17 + .../factory/data/VisitedObservationData.java | 17 + .../entity/ExperimentImportEntity.java | 17 + .../factory/entity/PendingDataset.java | 17 + .../factory/entity/PendingEntityFactory.java | 17 + .../factory/entity/PendingGermplasm.java | 17 + .../factory/entity/PendingLocation.java | 17 + .../factory/entity/PendingObservation.java | 17 + .../entity/PendingObservationUnit.java | 17 + .../factory/entity/PendingStudy.java | 17 + .../factory/entity/PendingTrial.java | 17 + .../AppendOverwriteIDValidation.java | 17 + .../middleware/commit/BrAPICommit.java | 17 + .../middleware/commit/BrAPIDatasetCommit.java | 17 + .../commit/BrAPIObservationCommit.java | 17 + .../commit/BrAPIObservationUnitCommit.java | 17 + .../middleware/commit/BrAPIStudyCommit.java | 17 + .../middleware/commit/BrAPITrialCommit.java | 17 + .../middleware/commit/LocationCommit.java | 17 + .../initialize/WorkflowInitialization.java | 17 + .../middleware/process/AppendStatistic.java | 17 + .../process/ImportTableProcess.java | 17 + .../model/AppendOverwriteMiddleware.java | 17 + .../AppendOverwriteMiddlewareContext.java | 17 + .../model/AppendOverwriteWorkflowContext.java | 17 + .../appendoverwrite/model/Middleware.java | 17 + .../model/MiddlewareException.java | 17 + .../workflow/NewExperimentWorkflow.java | 17 + .../experiment/service/DatasetService.java | 164 +--------- .../experiment/service/GermplasmService.java | 90 +----- .../experiment/service/LocationService.java | 129 +------- .../service/ObservationService.java | 251 +-------------- .../service/ObservationUnitService.java | 211 +------------ .../service/ObservationVariableService.java | 208 +------------ .../experiment/service/StatisticsService.java | 92 ------ .../experiment/service/StudyService.java | 206 +----------- .../experiment/service/TrialService.java | 101 +----- .../experiment/service/ValidateService.java | 292 ------------------ .../validator/field/DateValidator.java | 17 + .../validator/field/FieldValidator.java | 17 + .../validator/field/NominalValidator.java | 17 + .../validator/field/NumericalValidator.java | 17 + .../validator/field/ObservationValidator.java | 17 + .../validator/field/OrdinalValidator.java | 17 + 62 files changed, 1020 insertions(+), 1608 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StatisticsService.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java index c34794bab..d0f094cf2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.model.workflow; @FunctionalInterface diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java index f26342d3b..9be01f51b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.model.workflow; @FunctionalInterface diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java index 19be9bdcc..a0a81f123 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.model.workflow; @FunctionalInterface diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java index 17f48ee57..45af60609 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.model.workflow; import io.micronaut.core.order.Ordered; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 30eadea28..06f241b75 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment; import com.google.gson.Gson; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index b1ed662bb..fdb68a548 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment; import io.micronaut.context.annotation.Primary; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index c0dcaeb8b..8b6aa0032 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite; import lombok.Getter; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java index e21a0e7c8..c9ff1a1c0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; public interface BrAPIState { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java index 0f2af6bd2..9648d8379 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; import org.brapi.client.v2.model.exceptions.ApiException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java index fdbd41d25..2024ec07d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; import io.micronaut.context.annotation.Bean; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java index 30a9ae341..dabd5811c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; import io.micronaut.context.annotation.Bean; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java index 7e794b868..9c361d746 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; import io.micronaut.context.annotation.Bean; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java index 571f0b8d6..05ab4d096 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java index caeedd3cd..31e953df6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java index 95de8f367..61f80d711 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java index 5d5dc1403..7b631d49f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; import com.google.gson.Gson; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java index 93ca0f421..516527f14 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; import com.google.gson.Gson; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java index b744a3ba4..2c30b0bcb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; import io.micronaut.context.annotation.Bean; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UnchangedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UnchangedData.java index 9f59d3251..708d8eb50 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UnchangedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UnchangedData.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/VisitedObservationData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/VisitedObservationData.java index 13c1f1695..32a1b3d0b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/VisitedObservationData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/VisitedObservationData.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; import org.brapi.v2.model.pheno.BrAPIObservation; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java index 1f37a3425..c98d9cf83 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import org.brapi.client.v2.model.exceptions.ApiException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java index 2694e8974..3e8dd03e5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java index 9823d1f3c..20e827f2f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import io.micronaut.context.annotation.Bean; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java index f8009d37b..0076e686e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java index 537984844..538b0a7c3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java index e8ad7183b..bf15734e1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java index a829cc5df..6bb682ee2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java index 8774c3f7c..4397d205c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java index 7a43bf333..9c0c80d34 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java index adda352a6..14bad6915 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java index 97ce72522..84a9f7e63 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index a7ef0b53b..b7f92086f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java index 458369403..3010ad678 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index baa6eec48..c2b7533d0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java index b2ce28374..d3f8eae17 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index 1b62389e4..2b4373781 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index 0cb3e0f4a..7b72a37df 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index 3da2b847f..e65d11d15 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java index aeb88cfe8..9d9c628a6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; import io.micronaut.context.annotation.Prototype; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 3882785e6..6d80336b8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; import com.google.gson.Gson; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteMiddleware.java index 799b7847e..8e8478db5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteMiddleware.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteMiddlewareContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteMiddlewareContext.java index 33713d730..da868325b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteMiddlewareContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteMiddlewareContext.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; import lombok.Builder; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java index 2d4098612..13d892722 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; import lombok.Getter; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/Middleware.java index 2024efa0b..0a730a7b2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/Middleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/Middleware.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; public abstract class Middleware { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/MiddlewareException.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/MiddlewareException.java index 3b8b79c85..b35a5e246 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/MiddlewareException.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/MiddlewareException.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; import lombok.Getter; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java index 0b67766a5..92e1f6945 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; import lombok.Getter; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index a4cc47aad..f85137e15 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.context.annotation.Property; @@ -95,151 +112,4 @@ public PendingImportObject constructPIOFromDataset(BrAPIListDe // Create a PendingImportObject for the dataset with the existing list and reference ID return new PendingImportObject(ImportObjectState.EXISTING, dataset, UUID.fromString(xref.getReferenceId())); } - - - - // TODO: used by create workflow -// public Map> initializeObsVarDatasetByName(Program program, List experimentImportRows) { -// Map> obsVarDatasetByName = new HashMap<>(); -// -// Optional> trialPIO = getTrialPIO(experimentImportRows); -// -// if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { -// String datasetId = trialPIO.get().getBrAPIObject() -// .getAdditionalInfo() -// .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) -// .getAsString(); -// try { -// List existingDatasets = brAPIListDAO -// .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, -// program.getId(), -// String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), -// UUID.fromString(datasetId)); -// if (existingDatasets == null || existingDatasets.isEmpty()) { -// throw new InternalServerException("existing dataset summary not returned from brapi server"); -// } -// BrAPIListDetails dataSetDetails = brAPIListDAO -// .getListById(existingDatasets.get(0).getListDbId(), program.getId()) -// .getResult(); -// processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); -// } catch (ApiException e) { -// log.error(Utilities.generateApiExceptionLogMessage(e), e); -// throw new InternalServerException(e.toString(), e); -// } -// } -// return obsVarDatasetByName; -// } - - // TODO: used by expunit workflow - public Map> mapPendingObsDatasetByOUId( - String unitId, - Map> trialByOUId, - Map> obsVarDatasetByName, - Map> obsVarDatasetByOUId) { - if (!trialByOUId.isEmpty() && !obsVarDatasetByName.isEmpty() && - trialByOUId.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { - obsVarDatasetByOUId.put(unitId, obsVarDatasetByName.values().iterator().next()); - } - - return obsVarDatasetByOUId; - } - - // TODO: used by both workflows - public void addObsVarsToDatasetDetails(PendingImportObject pio, List referencedTraits, Program program) { - BrAPIListDetails details = pio.getBrAPIObject(); - referencedTraits.forEach(trait -> { - String id = Utilities.appendProgramKey(trait.getObservationVariableName(), program.getKey()); - - // TODO - Don't append the key if connected to a brapi service operating with legacy data(no appended program key) - - if (!details.getData().contains(id) && ImportObjectState.EXISTING != pio.getState()) { - details.getData().add(id); - } - if (!details.getData().contains(id) && ImportObjectState.EXISTING == pio.getState()) { - details.getData().add(id); - pio.setState(ImportObjectState.MUTATED); - } - }); - } - - - // TODO: used by create workflow -// public void fetchOrCreateDatasetPIO(ImportContext importContext, -// PendingData pendingData, -// List referencedTraits) throws UnprocessableEntityException { -// PendingImportObject pio; -// PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); -// -// String name = String.format("Observation Dataset [%s-%s]", -// program.getKey(), -// trialPIO.getBrAPIObject() -// .getAdditionalInfo() -// .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) -// .getAsString()); -// if (obsVarDatasetByName.containsKey(name)) { -// pio = obsVarDatasetByName.get(name); -// } else { -// UUID id = UUID.randomUUID(); -// BrAPIListDetails newDataset = importRow.constructDatasetDetails( -// name, -// id, -// BRAPI_REFERENCE_SOURCE, -// program, -// trialPIO.getId().toString()); -// pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); -// trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); -// if (ImportObjectState.EXISTING == trialPIO.getState()) { -// trialPIO.setState(ImportObjectState.MUTATED); -// } -// obsVarDatasetByName.put(name, pio); -// } -// addObsVarsToDatasetDetails(pio, referencedTraits, program); -// } - - // TODO: used by both workflows -// public List commitNewPendingDatasetsToBrAPIStore(ImportContext importContext, PendingData pendingData) { -// List newDatasetRequests = ProcessorData.getNewObjects(obsVarDatasetByName).stream().map(details -> { -// BrAPIListNewRequest request = new BrAPIListNewRequest(); -// request.setListName(details.getListName()); -// request.setListType(details.getListType()); -// request.setExternalReferences(details.getExternalReferences()); -// request.setAdditionalInfo(details.getAdditionalInfo()); -// request.data(details.getData()); -// return request; -// }).collect(Collectors.toList()); -// List createdDatasets = new ArrayList<>(brAPIListDAO.createBrAPILists(newDatasetRequests, program.getId(), upload)); -// for (BrAPIListSummary summary : createdDatasets) { -// obsVarDatasetByName.get(summary.getListName()).getBrAPIObject().setListDbId(summary.getListDbId()); -// } -// return createdDatasets; -// } - - // TODO: used by both workflows -// public List commitUpdatedPendingDatasetsToBrAPIStore(ImportContext importContext, PendingData pendingData) { -// List updatedDatasets = new ArrayList<>(); -// Map datasetNewDataById = ProcessorData -// .getMutationsByObjectId(obsVarDatasetByName, BrAPIListSummary::getListDbId); -// for (Map.Entry entry : datasetNewDataById.entrySet()) { -// String id = entry.getKey(); -// BrAPIListDetails dataset = entry.getValue(); -// try { -// List existingObsVarIds = brAPIListDAO.getListById(id, program.getId()).getResult().getData(); -// List newObsVarIds = dataset -// .getData() -// .stream() -// .filter(obsVarId -> !existingObsVarIds.contains(obsVarId)).collect(Collectors.toList()); -// List obsVarIds = new ArrayList<>(existingObsVarIds); -// obsVarIds.addAll(newObsVarIds); -// dataset.setData(obsVarIds); -// updatedDatasets.add(brAPIListDAO.updateBrAPIList(id, dataset, program.getId())); -// } catch (ApiException e) { -// log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); -// throw new InternalServerException("Error saving experiment import", e); -// } catch (Exception e) { -// log.error("Error updating dataset observation variables: ", e); -// throw new InternalServerException(e.getMessage(), e); -// } -// } -// return updatedDatasets; -// } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java index a6ffd6610..8ed3aa17e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.context.annotation.Property; @@ -78,77 +95,4 @@ public String getGIDFromGermplasmPIO(PendingImportObject pio) { gid = pio.getBrAPIObject().getAccessionNumber(); return gid; } - - // TODO: used by expunit workflow -// public Map> initializeGermplasmByGIDForExistingObservationUnits( -// Map> unitByName, -// Program program) { -// Map> existingGermplasmByGID = new HashMap<>(); -// -// List existingGermplasms = new ArrayList<>(); -// if(unitByName.size() > 0) { -// Set germplasmDbIds = unitByName.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); -// try { -// existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); -// } catch (ApiException e) { -// log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); -// throw new InternalServerException(e.toString(), e); -// } -// } -// -// existingGermplasms.forEach(existingGermplasm -> { -// BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) -// .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); -// existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); -// }); -// return existingGermplasmByGID; -// } - - // TODO: used by create worflow -// public Map> initializeExistingGermplasmByGID(Program program, List experimentImportRows) { -// Map> existingGermplasmByGID = new HashMap<>(); -// -// List existingGermplasms = new ArrayList<>(); -// if(observationUnitByNameNoScope.size() > 0) { -// Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); -// try { -// existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); -// } catch (ApiException e) { -// log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); -// throw new InternalServerException(e.toString(), e); -// } -// } -// -// List uniqueGermplasmGIDs = experimentImportRows.stream() -// .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) -// .map(ExperimentObservation::getGid) -// .distinct() -// .collect(Collectors.toList()); -// -// try { -// existingGermplasms.addAll(this.getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); -// } catch (ApiException e) { -// log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); -// throw new InternalServerException(e.toString(), e); -// } -// -// existingGermplasms.forEach(existingGermplasm -> { -// BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) -// .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); -// existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); -// }); -// return existingGermplasmByGID; -// } - - // TODO: used by expunit workflow - public Map> mapGermplasmByOUId( - String unitId, - BrAPIObservationUnit unit, - Map> germplasmByName, - Map> germplasmByOUId) { - String gid = unit.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.GID).getAsString(); - germplasmByOUId.put(unitId, germplasmByName.get(gid)); - - return germplasmByOUId; - } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java index 3305545c0..73466224f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.http.server.exceptions.InternalServerException; @@ -89,116 +106,4 @@ public Map> initializeLocationByNam ); return locationByName; } - - // TODO: used by create workflow -// public Map> initializeUniqueLocationNames(Program program, List experimentImportRows) { -// Map> locationByName = new HashMap<>(); -// -// List existingLocations = new ArrayList<>(); -// if(studyByNameNoScope.size() > 0) { -// Set locationDbIds = studyByNameNoScope.values() -// .stream() -// .map(study -> study.getBrAPIObject() -// .getLocationDbId()) -// .collect(Collectors.toSet()); -// try { -// existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); -// } catch (ApiException e) { -// log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); -// throw new InternalServerException(e.toString(), e); -// } -// } -// -// List uniqueLocationNames = experimentImportRows.stream() -// .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) -// .map(ExperimentObservation::getEnvLocation) -// .distinct() -// .filter(Objects::nonNull) -// .collect(Collectors.toList()); -// -// try { -// existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); -// } catch (ApiException e) { -// log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); -// throw new InternalServerException(e.toString(), e); -// } -// -// existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); -// return locationByName; -// } - - // TODO: used by expunit workflow - public Map> mapPendingLocationByOUId( - String unitId, - BrAPIObservationUnit unit, - Map> studyByOUId, - Map> locationByName, - Map> locationByOUId - ) { - if (unit.getLocationName() != null) { - locationByOUId.put(unitId, locationByName.get(unit.getLocationName())); - } else if (studyByOUId.get(unitId) != null && studyByOUId.get(unitId).getBrAPIObject().getLocationName() != null) { - locationByOUId.put( - unitId, - locationByName.get(studyByOUId.get(unitId).getBrAPIObject().getLocationName()) - ); - } else { - throw new IllegalStateException("Observation unit missing location: " + unitId); - } - - return locationByOUId; - } - - // TODO: used by expunit workflow -// private void fetchOrCreateLocationPIO(ImportContext importContext, ExpUnitContext expUnitContext) { -// PendingImportObject pio; -// String envLocationName = pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName(); -// if (!locationByName.containsKey((importRow.getEnvLocation()))) { -// ProgramLocation newLocation = new ProgramLocation(); -// newLocation.setName(envLocationName); -// pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); -// this.locationByName.put(envLocationName, pio); -// } -// } - - // TODO: used by create workflow -// private void fetchOrCreateLocationPIO(ImportContext importContext) { -// PendingImportObject pio; -// String envLocationName = importRow.getEnvLocation(); -// if (!locationByName.containsKey((importRow.getEnvLocation()))) { -// ProgramLocation newLocation = new ProgramLocation(); -// newLocation.setName(envLocationName); -// pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); -// this.locationByName.put(envLocationName, pio); -// } -// } - - // TODO: used by both workflows -// public List commitNewPendingLocationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { -// AuthenticatedUser actingUser = new AuthenticatedUser(upload.getUpdatedByUser().getName(), new ArrayList<>(), upload.getUpdatedByUser().getId(), new ArrayList<>()); -// -// List newLocations = ProcessorData.getNewObjects(this.locationByName) -// .stream() -// .map(location -> ProgramLocationRequest.builder() -// .name(location.getName()) -// .build()) -// .collect(Collectors.toList()); -// -// List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); -// // set the DbId to the for each newly created location -// for (ProgramLocation createdLocation : createdLocations) { -// String createdLocationName = createdLocation.getName(); -// this.locationByName.get(createdLocationName) -// .getBrAPIObject() -// .setLocationDbId(createdLocation.getLocationDbId()); -// } -// return createdLocations; -// } - - // TODO: used by both workflows -// public List commitUpdatedPendingLocationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { -// List updatedLocations = new ArrayList<>(); -// -// return updatedLocations; -// } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index 5bd097b8a..d9c808fed 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.service; import org.apache.commons.codec.digest.DigestUtils; @@ -125,238 +142,4 @@ public BrAPIObservation constructNewBrAPIObservation(boolean commit, } return observation; } -// public Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { -// Set ouDbIds = new HashSet<>(); -// Set variableDbIds = new HashSet<>(); -// Map variableNameByDbId = new HashMap<>(); -// Map ouNameByDbId = new HashMap<>(); -// Map studyNameByDbId = studyByNameNoScope.values() -// .stream() -// .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) -// .map(PendingImportObject::getBrAPIObject) -// .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); -// -// studyNameByDbId.keySet().forEach(studyDbId -> { -// try { -// brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { -// if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { -// ouDbIds.add(ou.getObservationUnitDbId()); -// } -// ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); -// }); -// } catch (ApiException e) { -// throw new RuntimeException(e); -// } -// }); -// -// for (Trait referencedTrait : referencedTraits) { -// variableDbIds.add(referencedTrait.getObservationVariableDbId()); -// variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); -// } -// -// List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); -// -// return existingObservations.stream() -// .map(obs -> { -// String studyName = studyNameByDbId.get(obs.getStudyDbId()); -// String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); -// String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); -// -// String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); -// -// return Map.entry(key, obs); -// }) -// .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); -// } - // TODO: used with expUnit workflow -// public void validateObservations(PendingData pendingData, -// int rowNum, -// ImportContext importContext, -// ExpUnitContext expUnitContext, -// List> phenotypeCols, -// CaseInsensitiveMap colVarMap) { -// for (Column phenoCol : phenotypeCols) { -// String importHash; -// String importObsValue = phenoCol.getString(rowNum); -// -// importHash = getImportObservationHash( -// pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), -// getVariableNameFromColumn(phenoCol), -// pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() -// ); -// -// validateObservation(importHash); -// } -// } - - // TODO: used with create workflow -// public void validateObservations(PendingData pendingData, -// int rowNum, -// ImportContext importContext, -// List> phenotypeCols, -// CaseInsensitiveMap colVarMap) { -// -// -// for (Column phenoCol : phenotypeCols) { -// String importHash; -// String importObsValue = phenoCol.getString(rowNum); -// -// importHash = getImportObservationHash(importRow, phenoCol.name()); -// -// validateObservation(importHash); -// } -// -// } - - // TODO: used by both workflows -// private void validateObservation(String importHash) { -// -// -// String importObsValue = phenoCol.getString(rowNum); -// -// -// // error if import observation data already exists and user has not selected to overwrite -// if (commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && -// this.existingObsByObsHash.containsKey(importHash) && -// StringUtils.isNotBlank(phenoCol.getString(rowNum)) && -// !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { -// addRowError( -// phenoCol.name(), -// String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), -// validationErrors, rowNum -// ); -// -// // preview case where observation has already been committed and the import row ObsVar data differs from what -// // had been saved prior to import -// } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { -// -// // add a change log entry when updating the value of an observation -// if (commit) { -// BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); -// DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); -// String timestamp = formatter.format(OffsetDateTime.now()); -// String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; -// String prior = ""; -// if (isValueMatched(importHash, importObsValue)) { -// prior.concat(existingObsByObsHash.get(importHash).getValue()); -// } -// if (timeStampColByPheno.containsKey(phenoCol.name()) && isTimestampMatched(importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { -// prior = prior.isEmpty() ? prior : prior.concat(" "); -// prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); -// } -// ChangeLogEntry change = new ChangeLogEntry(prior, -// reason, -// user.getId(), -// timestamp -// ); -// -// // create the changelog field in additional info if it does not already exist -// OverwrittenData.createChangeLog(pendingObservation); -// -// // add a new entry to the changelog -// pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); -// } -// -// // preview case where observation has already been committed and import ObsVar data is the -// // same as has been committed prior to import -// } else if (isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { -// BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); -// existingObs.setObservationVariableName(phenoCol.name()); -// observationByHash.get(importHash).setState(ImportObjectState.EXISTING); -// observationByHash.get(importHash).setBrAPIObject(existingObs); -// -// // preview case where observation has already been committed and import ObsVar data is empty prior to import -// } else if (!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { -// observationByHash.get(importHash).setState(ImportObjectState.EXISTING); -// } else { -// validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); -// -// //Timestamp validation -// if (timeStampColByPheno.containsKey(phenoCol.name())) { -// Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); -// validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); -// } -// } -// -// } - - // TODO: used by both workflows -// private void updateObservationDependencyValues(Program program) { -// String programKey = program.getKey(); -// -// // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds -// this.observationUnitByNameNoScope.values().stream() -// .map(PendingImportObject::getBrAPIObject) -// .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); -// -// // Update ObservationVariable DbIds -// List traits = getTraitList(program); -// CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); -// for ( Trait trait: traits) { -// traitMap.put(trait.getObservationVariableName(),trait); -// } -// for (PendingImportObject observation : this.observationByHash.values()) { -// String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); -// if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { -// String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); -// observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); -// } -// } -// } - - // TODO: used by both workflows -// public List commitNewPendingObservationsToBrAPIStore(ImportContext context, PendingData pendingData) { -// // filter out observations with no 'value' so they will not be saved -// List newObservations = ProcessorData.getNewObjects(this.observationByHash) -// .stream() -// .filter(obs -> !obs.getValue().isBlank()) -// .collect(Collectors.toList()); -// -// updateObservationDependencyValues(program); -// return brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); -// -// } - - // TODO: used by both workflows -// public List commitUpdatedPendingObservationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { -// List updatedObservations = new ArrayList<>(); -// Map mutatedObservationByDbId = ProcessorData -// .getMutationsByObjectId(observationByHash, BrAPIObservation::getObservationDbId); -// -// for (Map.Entry entry : mutatedObservationByDbId.entrySet()) { -// String id = entry.getKey(); -// BrAPIObservation observation = entry.getValue(); -// try { -// if (observation == null) { -// throw new Exception("Null observation"); -// } -// BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); -// updatedObservations.add(updatedObs); -// -// if (updatedObs == null) { -// throw new Exception("Null updated observation"); -// } -// -// if (!Objects.equals(observation.getValue(), updatedObs.getValue()) -// || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { -// String message; -// if (!Objects.equals(observation.getValue(), updatedObs.getValue())) { -// message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); -// } else { -// message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); -// } -// throw new Exception(message); -// } -// -// } catch (ApiException e) { -// log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); -// throw new InternalServerException("Error saving experiment import", e); -// } catch (Exception e) { -// log.error("Error updating observation: ", e); -// throw new InternalServerException(e.getMessage(), e); -// } -// } -// -// return updatedObservations; -// } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index 1b1e3479c..ff5da2813 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.context.annotation.Property; @@ -171,198 +188,4 @@ public List collectMissingOUIds(Set referenceIds, List fetchOrCreateObsUnitPIO(ImportContext importContext, -// PendingData pendingData, -// ExpUnitContext expUnitContext, -// String envSeqValue) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { -// PendingImportObject pio; -// String key = createObservationUnitKey(importRow); -// if (hasAllReferenceUnitIds) { -// pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); -// } else if (observationUnitByNameNoScope.containsKey(key)) { -// pio = observationUnitByNameNoScope.get(key); -// } else { -// String germplasmName = ""; -// if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { -// germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) -// .getBrAPIObject() -// .getGermplasmName(); -// } -// PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; -// UUID trialID = trialPIO.getId(); -// UUID datasetId = null; -// if (commit) { -// datasetId = UUID.fromString(trialPIO.getBrAPIObject() -// .getAdditionalInfo().getAsJsonObject() -// .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); -// } -// PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); -// UUID studyID = studyPIO.getId(); -// UUID id = UUID.randomUUID(); -// BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); -// -// // check for existing units if this is an existing study -// if (studyPIO.getBrAPIObject().getStudyDbId() != null) { -// List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); -// List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); -// if (matchingOU.isEmpty()) { -// throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); -// } else { -// pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); -// } -// } else { -// pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); -// } -// this.observationUnitByNameNoScope.put(key, pio); -// } -// return pio; -// } - - // TODO: used by create workflow -// public PendingImportObject fetchOrCreateObsUnitPIO(ImportContext importContext, -// PendingData pendingData, -// String envSeqValue) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { -// PendingImportObject pio; -// String key = createObservationUnitKey(importRow); -// if (hasAllReferenceUnitIds) { -// pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); -// } else if (observationUnitByNameNoScope.containsKey(key)) { -// pio = observationUnitByNameNoScope.get(key); -// } else { -// String germplasmName = ""; -// if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { -// germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) -// .getBrAPIObject() -// .getGermplasmName(); -// } -// PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; -// UUID trialID = trialPIO.getId(); -// UUID datasetId = null; -// if (commit) { -// datasetId = UUID.fromString(trialPIO.getBrAPIObject() -// .getAdditionalInfo().getAsJsonObject() -// .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); -// } -// PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); -// UUID studyID = studyPIO.getId(); -// UUID id = UUID.randomUUID(); -// BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); -// -// // check for existing units if this is an existing study -// if (studyPIO.getBrAPIObject().getStudyDbId() != null) { -// List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); -// List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); -// if (matchingOU.isEmpty()) { -// throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); -// } else { -// pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); -// } -// } else { -// pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); -// } -// this.observationUnitByNameNoScope.put(key, pio); -// } -// return pio; -// } - - // TODO: used by both workflows - public String createObservationUnitKey(ExperimentObservation importRow) { - return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); - } - - // TODO: used by both workflows - public String createObservationUnitKey(String studyName, String obsUnitName) { - return studyName + obsUnitName; - } - - // TODO: used by create workflow -// public void validateObservationUnits(ValidationErrors validationErrors, -// Set uniqueStudyAndObsUnit, -// int rowNum, -// ExperimentObservation importRow) { -// validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); -// -// String key = createObservationUnitKey(importRow); -// PendingImportObject ouPIO = observationUnitByNameNoScope.get(key); -// if(ouPIO.getState() == ImportObjectState.NEW && StringUtils.isNotBlank(importRow.getObsUnitID())) { -// addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "Could not find observation unit by ObsUnitDBID", validationErrors, rowNum); -// } -// -// validateGeoCoordinates(validationErrors, rowNum, importRow); -// } - - // TODO: used by both workflows -// private void updateObsUnitDependencyValues(String programKey) { -// -// // update study DbIds -// this.studyByNameNoScope.values() -// .stream() -// .filter(Objects::nonNull) -// .distinct() -// .map(PendingImportObject::getBrAPIObject) -// .forEach(study -> updateStudyDbId(study, programKey)); -// -// // update germplasm DbIds -// this.existingGermplasmByGID.values() -// .stream() -// .filter(Objects::nonNull) -// .distinct() -// .map(PendingImportObject::getBrAPIObject) -// .forEach(this::updateGermplasmDbId); -// } - - // TODO: used by both workflows -// public List commitNewPendingObservationUnitsToBrAPIStore(ImportContext context, PendingData pendingData) { -// List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); -// updateObsUnitDependencyValues(program.getKey()); -// List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); -// -// // set the DbId to the for each newly created Observation Unit -// for (BrAPIObservationUnit createdObservationUnit : createdObservationUnits) { -// // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope -// String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getStudyName(), program.getKey()); -// String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getObservationUnitName(), program.getKey()); -// String createdObsUnit_key = createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); -// this.observationUnitByNameNoScope.get(createdObsUnit_key) -// .getBrAPIObject() -// .setObservationUnitDbId(createdObservationUnit.getObservationUnitDbId()); -// } -// -// return createdObservationUnits; -// } - - // TODO: used by both workflows -// public List commitUpdatedPendingObservationUnitToBrAPIStore(ImportContext importContext, PendingData pendingData) { -// List updatedUnits = new ArrayList<>(); -// -// return updatedUnits; -// } -// -// private void updateStudyDbId(BrAPIStudy study, String programKey) { -// this.observationUnitByNameNoScope.values() -// .stream() -// .filter(obsUnit -> obsUnit.getBrAPIObject() -// .getStudyName() -// .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), programKey))) -// .forEach(obsUnit -> { -// obsUnit.getBrAPIObject() -// .setStudyDbId(study.getStudyDbId()); -// obsUnit.getBrAPIObject() -// .setTrialDbId(study.getTrialDbId()); -// }); -// } - -// private void updateGermplasmDbId(BrAPIGermplasm germplasm) { -// this.observationUnitByNameNoScope.values() -// .stream() -// .filter(obsUnit -> germplasm.getAccessionNumber() != null && -// germplasm.getAccessionNumber().equals(obsUnit -// .getBrAPIObject() -// .getAdditionalInfo().getAsJsonObject() -// .get(BrAPIAdditionalInfoFields.GID).getAsString())) -// .forEach(obsUnit -> obsUnit.getBrAPIObject() -// .setGermplasmDbId(germplasm.getGermplasmDbId())); -// } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java index 992c44a32..53bb9b745 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.http.HttpStatus; @@ -72,195 +89,4 @@ public Optional> validateMatchedTimestamps(Set obs return ve; } - // TODO: used with expUnit workflow -// public void validateObservations(PendingData pendingData, -// int rowNum, -// ImportContext importContext, -// ExpUnitContext expUnitContext, -// List> phenotypeCols, -// CaseInsensitiveMap colVarMap) { -// for (Column phenoCol : phenotypeCols) { -// String importHash; -// String importObsValue = phenoCol.getString(rowNum); -// -// importHash = getImportObservationHash( -// pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), -// getVariableNameFromColumn(phenoCol), -// pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() -// ); -// -// validateObservation(importHash); -// } -// } - - // TODO: used with create workflow -// public void validateObservations(PendingData pendingData, -// int rowNum, -// ImportContext importContext, -// List> phenotypeCols, -// CaseInsensitiveMap colVarMap) { -// -// -// for (Column phenoCol : phenotypeCols) { -// String importHash; -// String importObsValue = phenoCol.getString(rowNum); -// -// importHash = getImportObservationHash(importRow, phenoCol.name()); -// -// validateObservation(importHash); -// } -// -// } - - // TODO: used by both workflows -// private void validateObservation(String importHash) { -// -// -// String importObsValue = phenoCol.getString(rowNum); -// -// -// // error if import observation data already exists and user has not selected to overwrite -// if (commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && -// this.existingObsByObsHash.containsKey(importHash) && -// StringUtils.isNotBlank(phenoCol.getString(rowNum)) && -// !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { -// addRowError( -// phenoCol.name(), -// String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), -// validationErrors, rowNum -// ); -// -// // preview case where observation has already been committed and the import row ObsVar data differs from what -// // had been saved prior to import -// } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { -// -// // add a change log entry when updating the value of an observation -// if (commit) { -// BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); -// DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); -// String timestamp = formatter.format(OffsetDateTime.now()); -// String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; -// String prior = ""; -// if (isValueMatched(importHash, importObsValue)) { -// prior.concat(existingObsByObsHash.get(importHash).getValue()); -// } -// if (timeStampColByPheno.containsKey(phenoCol.name()) && isTimestampMatched(importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { -// prior = prior.isEmpty() ? prior : prior.concat(" "); -// prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); -// } -// ChangeLogEntry change = new ChangeLogEntry(prior, -// reason, -// user.getId(), -// timestamp -// ); -// -// // create the changelog field in additional info if it does not already exist -// OverwrittenData.createChangeLog(pendingObservation); -// -// // add a new entry to the changelog -// pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); -// } -// -// // preview case where observation has already been committed and import ObsVar data is the -// // same as has been committed prior to import -// } else if (isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { -// BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); -// existingObs.setObservationVariableName(phenoCol.name()); -// observationByHash.get(importHash).setState(ImportObjectState.EXISTING); -// observationByHash.get(importHash).setBrAPIObject(existingObs); -// -// // preview case where observation has already been committed and import ObsVar data is empty prior to import -// } else if (!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { -// observationByHash.get(importHash).setState(ImportObjectState.EXISTING); -// } else { -// validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); -// -// //Timestamp validation -// if (timeStampColByPheno.containsKey(phenoCol.name())) { -// Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); -// validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); -// } -// } -// -// } - - // TODO: used by both workflows -// private void updateObservationDependencyValues(Program program) { -// String programKey = program.getKey(); -// -// // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds -// this.observationUnitByNameNoScope.values().stream() -// .map(PendingImportObject::getBrAPIObject) -// .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); -// -// // Update ObservationVariable DbIds -// List traits = getTraitList(program); -// CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); -// for ( Trait trait: traits) { -// traitMap.put(trait.getObservationVariableName(),trait); -// } -// for (PendingImportObject observation : this.observationByHash.values()) { -// String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); -// if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { -// String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); -// observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); -// } -// } -// } - - // TODO: used by both workflows -// public List commitNewPendingObservationsToBrAPIStore(ImportContext context, PendingData pendingData) { -// // filter out observations with no 'value' so they will not be saved -// List newObservations = ProcessorData.getNewObjects(this.observationByHash) -// .stream() -// .filter(obs -> !obs.getValue().isBlank()) -// .collect(Collectors.toList()); -// -// updateObservationDependencyValues(program); -// return brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); -// -// } - - // TODO: used by both workflows -// public List commitUpdatedPendingObservationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { -// List updatedObservations = new ArrayList<>(); -// Map mutatedObservationByDbId = ProcessorData -// .getMutationsByObjectId(observationByHash, BrAPIObservation::getObservationDbId); -// -// for (Map.Entry entry : mutatedObservationByDbId.entrySet()) { -// String id = entry.getKey(); -// BrAPIObservation observation = entry.getValue(); -// try { -// if (observation == null) { -// throw new Exception("Null observation"); -// } -// BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); -// updatedObservations.add(updatedObs); -// -// if (updatedObs == null) { -// throw new Exception("Null updated observation"); -// } -// -// if (!Objects.equals(observation.getValue(), updatedObs.getValue()) -// || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { -// String message; -// if (!Objects.equals(observation.getValue(), updatedObs.getValue())) { -// message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); -// } else { -// message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); -// } -// throw new Exception(message); -// } -// -// } catch (ApiException e) { -// log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); -// throw new InternalServerException("Error saving experiment import", e); -// } catch (Exception e) { -// log.error("Error updating observation: ", e); -// throw new InternalServerException(e.getMessage(), e); -// } -// } -// -// return updatedObservations; -// } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StatisticsService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StatisticsService.java deleted file mode 100644 index 0867cb81e..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StatisticsService.java +++ /dev/null @@ -1,92 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.service; - -import org.apache.commons.lang3.StringUtils; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; -import org.breedinginsight.brapps.importer.model.imports.PendingImport; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; -import org.breedinginsight.services.exceptions.ValidatorException; - -import java.util.HashSet; -import java.util.List; -import java.util.Map; - -public class StatisticsService { - // TODO: used by both workflows -// public Map generateStatisticsMap(List importRows) { -// // Data for stats. -// HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names -// HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's -// HashSet gidCounter = new HashSet<>(); // set of unique GID's -// -// for (BrAPIImport row : importRows) { -// ExperimentObservation importRow = (ExperimentObservation) row; -// // Collect date for stats. -// addIfNotNull(environmentNameCounter, importRow.getEnv()); -// addIfNotNull(obsUnitsIDCounter, createObservationUnitKey(importRow)); -// addIfNotNull(gidCounter, importRow.getGid()); -// } -// -// int numNewObservations = Math.toIntExact( -// observationByHash.values() -// .stream() -// .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && -// !StringUtils.isBlank(preview.getBrAPIObject() -// .getValue())) -// .count() -// ); -// -// int numExistingObservations = Math.toIntExact( -// this.observationByHash.values() -// .stream() -// .filter(preview -> preview != null && preview.getState() == ImportObjectState.EXISTING && -// !StringUtils.isBlank(preview.getBrAPIObject() -// .getValue())) -// .count() -// ); -// -// int numMutatedObservations = Math.toIntExact( -// this.observationByHash.values() -// .stream() -// .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED && -// !StringUtils.isBlank(preview.getBrAPIObject() -// .getValue())) -// .count() -// ); -// -// -// ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() -// .newObjectCount(environmentNameCounter.size()) -// .build(); -// ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() -// .newObjectCount(obsUnitsIDCounter.size()) -// .build(); -// ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() -// .newObjectCount(gidCounter.size()) -// .build(); -// ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder() -// .newObjectCount(numNewObservations) -// .build(); -// ImportPreviewStatistics existingObservationStats = ImportPreviewStatistics.builder() -// .newObjectCount(numExistingObservations) -// .build(); -// ImportPreviewStatistics mutatedObservationStats = ImportPreviewStatistics.builder() -// .newObjectCount(numMutatedObservations) -// .build(); -// -// return Map.of( -// "Environments", environmentStats, -// "Observation_Units", obdUnitStats, -// "GIDs", gidStats, -// "Observations", observationStats, -// "Existing_Observations", existingObservationStats, -// "Mutated_Observations", mutatedObservationStats -// ); -// } - - // TODO: used by both workflows - public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { - // TODO - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index fb21be4cc..06f6a3bbb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.context.annotation.Property; @@ -236,193 +253,4 @@ public String getStudyDbIdBelongingToPendingUnit(PendingImportObject> studyByName -// ) throws Exception { -// Set studyDbIds = observationUnitByNameNoScope.values() -// .stream() -// .map(pio -> pio.getBrAPIObject() -// .getStudyDbId()) -// .collect(Collectors.toSet()); -// -// List studies = fetchStudiesByDbId(studyDbIds, program); -// for (BrAPIStudy study : studies) { -// processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); -// } -// } - -// public Map> mapPendingStudyByOUId( -// String unitId, -// BrAPIObservationUnit unit, -// Map> studyByName, -// Map> studyByOUId, -// Program program -// ) { -// if (unit.getStudyName() != null) { -// String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); -// studyByOUId.put(unitId, studyByName.get(studyName)); -// } else { -// throw new IllegalStateException("Observation unit missing study name: " + unitId); -// } -// -// return studyByOUId; -// } - - // TODO: used by expunit workflow -// private PendingImportObject fetchOrCreateStudyPIO( -// ImportContext importContext, -// ExpUnitContext expUnitContext, -// String expSequenceValue, -// Supplier envNextVal -// ) throws UnprocessableEntityException { -// PendingImportObject pio; -// if (hasAllReferenceUnitIds) { -// String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( -// pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), -// program.getKey() -// ); -// pio = studyByNameNoScope.get(studyName); -// if (!commit){ -// addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); -// } -// } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { -// pio = studyByNameNoScope.get(importRow.getEnv()); -// if (!commit){ -// addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); -// } -// } else { -// PendingImportObject trialPIO = hasAllReferenceUnitIds ? -// getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); -// UUID trialID = trialPIO.getId(); -// UUID id = UUID.randomUUID(); -// BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); -// newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies -// -// // It is assumed that the study has only one season, And that the Years and not -// // the dbId's are stored in getSeason() list. -// String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season -// if (commit) { -// if(StringUtils.isNotBlank(year)) { -// String seasonID = this.yearToSeasonDbId(year, program.getId()); -// newStudy.setSeasons(Collections.singletonList(seasonID)); -// } -// } else { -// addYearToStudyAdditionalInfo(program, newStudy, year); -// } -// -// pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); -// this.studyByNameNoScope.put(importRow.getEnv(), pio); -// } -// return pio; -// } - - // TODO: used by create workflow -// private PendingImportObject fetchOrCreateStudyPIO( -// ImportContext importContext, -// String expSequenceValue, -// Supplier envNextVal -// ) throws UnprocessableEntityException { -// PendingImportObject pio; -// if (hasAllReferenceUnitIds) { -// String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( -// pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), -// program.getKey() -// ); -// pio = studyByNameNoScope.get(studyName); -// if (!commit){ -// addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); -// } -// } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { -// pio = studyByNameNoScope.get(importRow.getEnv()); -// if (!commit){ -// addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); -// } -// } else { -// PendingImportObject trialPIO = hasAllReferenceUnitIds ? -// getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); -// UUID trialID = trialPIO.getId(); -// UUID id = UUID.randomUUID(); -// BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); -// newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies -// -// // It is assumed that the study has only one season, And that the Years and not -// // the dbId's are stored in getSeason() list. -// String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season -// if (commit) { -// if(StringUtils.isNotBlank(year)) { -// String seasonID = this.yearToSeasonDbId(year, program.getId()); -// newStudy.setSeasons(Collections.singletonList(seasonID)); -// } -// } else { -// addYearToStudyAdditionalInfo(program, newStudy, year); -// } -// -// pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); -// this.studyByNameNoScope.put(importRow.getEnv(), pio); -// } -// return pio; -// } - -// private void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { -// // update location DbIds in studies for all distinct locations -// mappedBrAPIImport.values() -// .stream() -// .map(PendingImport::getLocation) -// .forEach(this::updateStudyLocationDbId); -// -// // update trial DbIds in studies for all distinct trials -// this.trialByNameNoScope.values() -// .stream() -// .filter(Objects::nonNull) -// .distinct() -// .map(PendingImportObject::getBrAPIObject) -// .forEach(trial -> this.updateTrialDbId(trial, programKey)); -// } -// -// private void updateStudyLocationDbId(PendingImportObject location) { -// this.studyByNameNoScope.values() -// .stream() -// .filter(study -> location.getId().toString() -// .equals(study.getBrAPIObject() -// .getLocationDbId())) -// .forEach(study -> study.getBrAPIObject() -// .setLocationDbId(location.getBrAPIObject().getLocationDbId())); -// } -// -// private void updateTrialDbId(BrAPITrial trial, String programKey) { -// this.studyByNameNoScope.values() -// .stream() -// .filter(study -> study.getBrAPIObject() -// .getTrialName() -// .equals(Utilities.removeProgramKey(trial.getTrialName(), programKey))) -// .forEach(study -> study.getBrAPIObject() -// .setTrialDbId(trial.getTrialDbId())); -// } - - // TODO: used by both workflows -// public List commitNewPendingStudiessToBrAPIStore(ImportContext context, PendingData pendingData) { -// List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); -// updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); -// List createdStudies = brAPIStudyDAO.createBrAPIStudies(newStudies, program.getId(), upload); -// -// // set the DbId to the for each newly created study -// for (BrAPIStudy createdStudy : createdStudies) { -// String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData(createdStudy.getStudyName(), program.getKey()); -// this.studyByNameNoScope.get(createdStudy_name_no_key) -// .getBrAPIObject() -// .setStudyDbId(createdStudy.getStudyDbId()); -// } -// -// return createdStudies; -// } - - // TODO: used by both workflows - public List commitUpdatedPendingStudiesToBrAPIStore(ImportContext importContext, PendingData pendingData) { - List updatedStudies = new ArrayList<>(); - - return updatedStudies; - } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index 119c67ed4..dcbc79159 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.context.annotation.Property; @@ -350,88 +367,4 @@ public Map> mapPendingTrialByOUId( return trialByOUId; } - - // TODO: overloaded method used by expunit workflow -// public PendingImportObject fetchOrCreateTrialPIO( -// ImportContext importContext, -// PendingData pendingData, -// ExpUnitContext expUnitContext -// ) throws UnprocessableEntityException { -// PendingImportObject trialPio; -// -// -// -// trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); -// -// -// return trialPio; -// } - - // TODO: overloaded method used by create workflow -// public PendingImportObject fetchOrCreateTrialPIO( -// ImportContext importContext, -// PendingData pendingData -// ) throws UnprocessableEntityException { -// PendingImportObject trialPio; -// -// -// if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { -// PendingImportObject envPio; -// trialPio = trialByNameNoScope.get(importRow.getExpTitle()); -// envPio = studyByNameNoScope.get(importRow.getEnv()); -// -// // creating new units for existing experiments and environments is not possible -// if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && -// (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ -// throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); -// } -// } else if (!trialByNameNoScope.isEmpty()) { -// throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); -// } else { -// UUID id = UUID.randomUUID(); -// String expSeqValue = null; -// if (commit) { -// expSeqValue = expNextVal.get().toString(); -// } -// BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); -// trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); -// trialByNameNoScope.put(importRow.getExpTitle(), trialPio); -// } -// -// return trialPio; -// } - - // TODO: used by both workflows -// public List commitNewPendingTrialsToBrAPIStore(ImportContext context, PendingData pendingData) { -// List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); -// List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newTrials, program.getId(), upload)); -// // set the DbId to the for each newly created trial -// for (BrAPITrial createdTrial : createdTrials) { -// String createdTrialName = Utilities.removeProgramKey(createdTrial.getTrialName(), program.getKey()); -// this.trialByNameNoScope.get(createdTrialName) -// .getBrAPIObject() -// .setTrialDbId(createdTrial.getTrialDbId()); -// } -// return createdTrials; -// } - -// public List commitUpdatedPendingTrialsToBrAPIStore(ImportContext importContext, PendingData pendingData) { -// List updatedTrials = new ArrayList<>(); -// Map mutatedTrialsById = ProcessorData -// .getMutationsByObjectId(trialByNameNoScope, BrAPITrial::getTrialDbId); -// for (Map.Entry entry : mutatedTrialsById.entrySet()) { -// String id = entry.getKey(); -// BrAPITrial trial = entry.getValue(); -// try { -// updatedTrials.add(brapiTrialDAO.updateBrAPITrial(id, trial, program.getId())); -// } catch (ApiException e) { -// log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); -// throw new InternalServerException("Error saving experiment import", e); -// } catch (Exception e) { -// log.error("Error updating dataset observation variables: ", e); -// throw new InternalServerException(e.getMessage(), e); -// } -// } -// return updatedTrials; -// } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java deleted file mode 100644 index 582eb78bf..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java +++ /dev/null @@ -1,292 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.service; - -public class ValidateService { - // TODO: used by expUnit workflow -// public void prepareDataForValidation(ImportContext importContext, -// ExpUnitContext expUnitContext, -// List> phenotypeCols) { -// for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { -// ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); -// PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); -// List> observations = mappedImportRow.getObservations(); -// String observationHash; -// if (hasAllReferenceUnitIds) { -// String refOUId = importRow.getObsUnitID(); -// mappedImportRow.setTrial(pendingTrialByOUId.get(refOUId)); -// mappedImportRow.setLocation(pendingLocationByOUId.get(refOUId)); -// mappedImportRow.setStudy(pendingStudyByOUId.get(refOUId)); -// mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(refOUId)); -// mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(refOUId)); -// -// // loop over phenotype column observation data for current row -// for (Column column : phenotypeCols) { -// observationHash = getObservationHash( -// pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + -// pendingObsUnitByOUId.get(refOUId).getBrAPIObject().getObservationUnitName(), -// getVariableNameFromColumn(column), -// pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() -// ); -// -// // if value was blank won't be entry in map for this observation -// observations.add(observationByHash.get(observationHash)); -// } -// -// } else { -// mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); -// mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); -// mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); -// mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); -// mappedImportRow.setGermplasm(getGidPIO(importRow)); -// -// // loop over phenotype column observation data for current row -// for (Column column : phenotypeCols) { -// -// // if value was blank won't be entry in map for this observation -// observations.add(observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); -// } -// } -// -// mappedBrAPIImport.put(rowNum, mappedImportRow); -// } -// } - - // TODO: used by create workflow -// public void prepareDataForValidation(ImportContext importContext, -// List> phenotypeCols) { -// for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { -// ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); -// PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); -// List> observations = mappedImportRow.getObservations(); -// String observationHash; -// if (hasAllReferenceUnitIds) { -// String refOUId = importRow.getObsUnitID(); -// mappedImportRow.setTrial(pendingTrialByOUId.get(refOUId)); -// mappedImportRow.setLocation(pendingLocationByOUId.get(refOUId)); -// mappedImportRow.setStudy(pendingStudyByOUId.get(refOUId)); -// mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(refOUId)); -// mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(refOUId)); -// -// // loop over phenotype column observation data for current row -// for (Column column : phenotypeCols) { -// observationHash = getObservationHash( -// pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + -// pendingObsUnitByOUId.get(refOUId).getBrAPIObject().getObservationUnitName(), -// getVariableNameFromColumn(column), -// pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() -// ); -// -// // if value was blank won't be entry in map for this observation -// observations.add(observationByHash.get(observationHash)); -// } -// -// } else { -// mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); -// mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); -// mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); -// mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); -// mappedImportRow.setGermplasm(getGidPIO(importRow)); -// -// // loop over phenotype column observation data for current row -// for (Column column : phenotypeCols) { -// -// // if value was blank won't be entry in map for this observation -// observations.add(observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); -// } -// } -// -// mappedBrAPIImport.put(rowNum, mappedImportRow); -// } -// } - - // TODO: used by expUnit workflow -// public void validateFields(ImportContext importContext, -// PendingData pendingData, -// ExpUnitContext expUnitContext, -// List referencedTraits, Program program, -// List> phenotypeCols) { -// //fetching any existing observations for any OUs in the import -// CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); -// for ( Trait trait: referencedTraits) { -// colVarMap.put(trait.getObservationVariableName(),trait); -// } -// Set uniqueStudyAndObsUnit = new HashSet<>(); -// for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { -// ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); -// PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); -// if (hasAllReferenceUnitIds) { -// validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); -// } else { -// if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. -// validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); -// } -// validateTestOrCheck(importRow, validationErrors, rowNum); -// validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); -// validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); -// validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); -// } -// } -// } - - // TODO: used by create workflow -// public void validateFields(ImportContext importContext, -// PendingData pendingData, -// List referencedTraits, Program program, -// List> phenotypeCols) { -// //fetching any existing observations for any OUs in the import -// CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); -// for ( Trait trait: referencedTraits) { -// colVarMap.put(trait.getObservationVariableName(),trait); -// } -// Set uniqueStudyAndObsUnit = new HashSet<>(); -// for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { -// ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); -// PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); -// if (hasAllReferenceUnitIds) { -// validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); -// } else { -// if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. -// validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); -// } -// validateTestOrCheck(importRow, validationErrors, rowNum); -// validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); -// validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); -// validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); -// } -// } -// } - - // TODO: used by create workflow -// private void validateTestOrCheck(ExperimentObservation importRow, ValidationErrors validationErrors, int rowNum) { -// String testOrCheck = importRow.getTestOrCheck(); -// if ( ! ( testOrCheck==null || testOrCheck.isBlank() -// || "C".equalsIgnoreCase(testOrCheck) || "CHECK".equalsIgnoreCase(testOrCheck) -// || "T".equalsIgnoreCase(testOrCheck) || "TEST".equalsIgnoreCase(testOrCheck) ) -// ){ -// addRowError(ExperimentObservation.Columns.TEST_CHECK, String.format("Invalid value (%s)", testOrCheck), validationErrors, rowNum) ; -// } -// } - - // TODO: used by create workflow -// private void validateConditionallyRequired(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, Program program, boolean commit) { -// ImportObjectState expState = this.trialByNameNoScope.get(importRow.getExpTitle()) -// .getState(); -// ImportObjectState envState = this.studyByNameNoScope.get(importRow.getEnv()).getState(); -// -// String errorMessage = BLANK_FIELD_EXPERIMENT; -// if (expState == ImportObjectState.EXISTING && envState == ImportObjectState.NEW) { -// errorMessage = BLANK_FIELD_ENV; -// } else if(expState == ImportObjectState.EXISTING && envState == ImportObjectState.EXISTING) { -// errorMessage = BLANK_FIELD_OBS; -// } -// -// if(expState == ImportObjectState.NEW || envState == ImportObjectState.NEW) { -// validateRequiredCell(importRow.getGid(), ExperimentObservation.Columns.GERMPLASM_GID, errorMessage, validationErrors, rowNum); -// validateRequiredCell(importRow.getExpTitle(), ExperimentObservation.Columns.EXP_TITLE,errorMessage, validationErrors, rowNum); -// validateRequiredCell(importRow.getExpUnit(), ExperimentObservation.Columns.EXP_UNIT, errorMessage, validationErrors, rowNum); -// validateRequiredCell(importRow.getExpType(), ExperimentObservation.Columns.EXP_TYPE, errorMessage, validationErrors, rowNum); -// validateRequiredCell(importRow.getEnv(), ExperimentObservation.Columns.ENV, errorMessage, validationErrors, rowNum); -// if(validateRequiredCell(importRow.getEnvLocation(), ExperimentObservation.Columns.ENV_LOCATION, errorMessage, validationErrors, rowNum)) { -// if(!Utilities.removeProgramKeyAndUnknownAdditionalData(this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getLocationName(), program.getKey()).equals(importRow.getEnvLocation())) { -// addRowError(ExperimentObservation.Columns.ENV_LOCATION, ENV_LOCATION_MISMATCH, validationErrors, rowNum); -// } -// } -// if(validateRequiredCell(importRow.getEnvYear(), ExperimentObservation.Columns.ENV_YEAR, errorMessage, validationErrors, rowNum)) { -// String studyYear = StringUtils.defaultString( this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getSeasons().get(0) ); -// String rowYear = importRow.getEnvYear(); -// if(commit) { -// rowYear = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); -// } -// if(StringUtils.isNotBlank(studyYear) && !studyYear.equals(rowYear)) { -// addRowError(ExperimentObservation.Columns.ENV_YEAR, ENV_YEAR_MISMATCH, validationErrors, rowNum); -// } -// } -// validateRequiredCell(importRow.getExpUnitId(), ExperimentObservation.Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); -// validateRequiredCell(importRow.getExpReplicateNo(), ExperimentObservation.Columns.REP_NUM, errorMessage, validationErrors, rowNum); -// validateRequiredCell(importRow.getExpBlockNo(), ExperimentObservation.Columns.BLOCK_NUM, errorMessage, validationErrors, rowNum); -// -// if(StringUtils.isNotBlank(importRow.getObsUnitID())) { -// addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); -// } -// } else { -// //Check if existing environment. If so, ObsUnitId must be assigned -// validateRequiredCell( -// importRow.getObsUnitID(), -// ExperimentObservation.Columns.OBS_UNIT_ID, -// MISSING_OBS_UNIT_ID_ERROR, -// validationErrors, -// rowNum -// ); -// } -// } - - // TODO: used by create workflow -// public void validateGeoCoordinates(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow) { -// -// String lat = importRow.getLatitude(); -// String lon = importRow.getLongitude(); -// String elevation = importRow.getElevation(); -// -// // If any of Lat, Long, or Elevation are provided, Lat and Long must both be provided. -// if (StringUtils.isNotBlank(lat) || StringUtils.isNotBlank(lon) || StringUtils.isNotBlank(elevation)) { -// if (StringUtils.isBlank(lat)) { -// addRowError(ExperimentObservation.Columns.LAT, "Latitude must be provided for complete coordinate specification", validationErrors, rowNum); -// } -// if (StringUtils.isBlank(lon)) { -// addRowError(ExperimentObservation.Columns.LONG, "Longitude must be provided for complete coordinate specification", validationErrors, rowNum); -// } -// } -// -// // Validate coordinate values -// boolean latBadValue = false; -// boolean lonBadValue = false; -// boolean elevationBadValue = false; -// double latDouble; -// double lonDouble; -// double elevationDouble; -// -// // Only check latitude format if not blank since already had previous error -// if (StringUtils.isNotBlank(lat)) { -// try { -// latDouble = Double.parseDouble(lat); -// if (latDouble < -90 || latDouble > 90) { -// latBadValue = true; -// } -// } catch (NumberFormatException e) { -// latBadValue = true; -// } -// } -// -// // Only check longitude format if not blank since already had previous error -// if (StringUtils.isNotBlank(lon)) { -// try { -// lonDouble = Double.parseDouble(lon); -// if (lonDouble < -180 || lonDouble > 180) { -// lonBadValue = true; -// } -// } catch (NumberFormatException e) { -// lonBadValue = true; -// } -// } -// -// if (StringUtils.isNotBlank(elevation)) { -// try { -// elevationDouble = Double.parseDouble(elevation); -// } catch (NumberFormatException e) { -// elevationBadValue = true; -// } -// } -// -// if (latBadValue) { -// addRowError(ExperimentObservation.Columns.LAT, "Invalid Lat value (expected range -90 to 90)", validationErrors, rowNum); -// } -// -// if (lonBadValue) { -// addRowError(ExperimentObservation.Columns.LONG, "Invalid Long value (expected range -180 to 180)", validationErrors, rowNum); -// } -// -// if (elevationBadValue) { -// addRowError(ExperimentObservation.Columns.LONG, "Invalid Elevation value (numerals expected)", validationErrors, rowNum); -// } -// -// } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java index 21b0e8448..594de233c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; import io.micronaut.http.HttpStatus; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/FieldValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/FieldValidator.java index 01253ef97..2747d9749 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/FieldValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/FieldValidator.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; import io.micronaut.context.annotation.Primary; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NominalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NominalValidator.java index bd2070002..ab390d2bf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NominalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NominalValidator.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; import io.micronaut.http.HttpStatus; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java index 5a4eba21d..d4498311a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; import io.micronaut.http.HttpStatus; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/ObservationValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/ObservationValidator.java index b28528316..6a5bb5e10 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/ObservationValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/ObservationValidator.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; import io.micronaut.core.annotation.NonNull; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/OrdinalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/OrdinalValidator.java index 4a3077077..4ea8bdead 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/OrdinalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/OrdinalValidator.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; import io.micronaut.http.HttpStatus; From 98cf134702f2c07a20620c190fb5b8fca4e03cb6 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 24 Jun 2024 00:06:10 -0400 Subject: [PATCH 162/203] optimize imports --- .../appendoverwrite/AppendOverwritePhenotypesWorkflow.java | 2 +- .../experiment/appendoverwrite/factory/data/InitialData.java | 3 +-- .../appendoverwrite/factory/data/OverwrittenData.java | 1 - .../experiment/appendoverwrite/factory/data/UnchangedData.java | 1 - .../experiment/create/workflow/NewExperimentWorkflow.java | 2 ++ 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 8b6aa0032..d6858ae9d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -159,7 +159,7 @@ public Optional process(ImportServiceContext context) { @Override public int getOrder() { - return 2; + return 20; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java index 7b631d49f..b74a03aa1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java @@ -26,11 +26,10 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.VisitedObservationData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; -import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; import org.breedinginsight.model.User; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java index 516527f14..3955d567b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java @@ -28,7 +28,6 @@ import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.VisitedObservationData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; import org.breedinginsight.model.Program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UnchangedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UnchangedData.java index 708d8eb50..4b1e6b92f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UnchangedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UnchangedData.java @@ -22,7 +22,6 @@ import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.VisitedObservationData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java index 92e1f6945..5d847dce2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java @@ -82,6 +82,8 @@ public Optional process(ImportServiceContext context) { // Build and return the preview response try { ImportPreviewResponse successResponse; + + // TODO: replace ProcessorManager#process with new-experiment workflow process from BI-2132 successResponse = processorManagerProvider.get().process(context.getBrAPIImports(), List.of(experimentProcessorProvider.get()), context.getData(), From 3a76d1fdb8dc652bb330c4ec4f3e918dacd576b9 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 24 Jun 2024 10:10:30 -0400 Subject: [PATCH 163/203] fix numerical validator to check for non-numeric values --- .../brapps/importer/model/imports/DomainImportService.java | 7 ++++--- .../appendoverwrite/AppendOverwritePhenotypesWorkflow.java | 4 ++-- .../middleware/process/ImportTableProcess.java | 2 +- .../experiment/validator/field/NumericalValidator.java | 5 +++++ 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 15c839799..ef27af1d6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -71,9 +71,10 @@ public ImportPreviewResponse process(ImportServiceContext context) Optional result = workflowNavigator.process(context); // Throw any exceptions caught during workflow processing - result.flatMap(ImportWorkflowResult::getCaughtException).ifPresent(error -> { - throw new RuntimeException(error.getCause()); - }); + if (result.flatMap(ImportWorkflowResult::getCaughtException).isPresent()) { + throw result.flatMap(ImportWorkflowResult::getCaughtException).get(); + } + return result.flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); } else { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index d6858ae9d..cf2e2ff0f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -110,7 +110,7 @@ public Optional process(ImportServiceContext context) { // Stop and return any errors that occurred while processing Optional previewException = Optional.ofNullable(processedPreviewContext.getAppendOverwriteWorkflowContext().getProcessError()); if (previewException.isPresent() ) { - log.debug(String.format("%s in %s", previewException.get().getException().getClass()), previewException.get().getLocalTransactionName()); + log.debug(String.format("%s in %s", previewException.get().getException().getClass().getName(), previewException.get().getLocalTransactionName())); result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.ofNullable(previewException.get().getException()))); return result; } @@ -159,7 +159,7 @@ public Optional process(ImportServiceContext context) { @Override public int getOrder() { - return 20; + return 2; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 6d80336b8..ddd6e9100 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -311,7 +311,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext } // Validate processed data - processedData.getValidationErrors().ifPresent(errList -> errList.forEach(e->validationErrors.addError(rowNum, e))); + processedData.getValidationErrors().ifPresent(errList -> errList.forEach(e->validationErrors.addError(rowNum + 2, e))); // +2 to account for header row and excel file 1-based row index if (validationErrors.hasErrors()) { throw new ValidatorException(validationErrors); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java index d4498311a..d3d2b74ce 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java @@ -67,7 +67,12 @@ public Optional validateField(String fieldName, String value, T return Optional.empty(); } + // Make new validation error if the value is non-numeric Optional number = observationService.validNumericValue(value); + if (number.isEmpty()) { + return Optional.of(new ValidationError(fieldName, "Non-numeric text in a numerical field", HttpStatus.UNPROCESSABLE_ENTITY)); + } + Optional validationError = number .flatMap(num -> { if (observationService.validNumericRange(num, variable.getScale())) { From b247b5d117d0c0e50f6ea442fa837fe31fa9bab4 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 24 Jun 2024 17:12:58 -0400 Subject: [PATCH 164/203] parse OffssetDateTime for yyyy-MM-dd format --- .../factory/data/OverwrittenData.java | 10 +++++++--- .../factory/data/ProcessedDataFactory.java | 7 ++++--- .../process/ImportTableProcess.java | 16 +++++++++------- .../service/ObservationService.java | 19 +++++++++++++++++++ .../validator/field/DateValidator.java | 2 +- 5 files changed, 40 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java index 3955d567b..308658158 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java @@ -29,6 +29,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; @@ -47,6 +48,7 @@ public class OverwrittenData extends VisitedObservationData { FieldValidator fieldValidator; + ObservationService observationService; Gson gson; boolean canOverwrite; boolean isCommit; @@ -74,7 +76,8 @@ public OverwrittenData(boolean canOverwrite, BrAPIObservation observation, UUID userId, Program program, - FieldValidator fieldValidator) { + FieldValidator fieldValidator, + ObservationService observationService) { this.canOverwrite = canOverwrite; this.isCommit = isCommit; this.unitId = unitId; @@ -88,6 +91,7 @@ public OverwrittenData(boolean canOverwrite, this.userId = userId; this.program = program; this.fieldValidator = fieldValidator; + this.observationService = observationService; this.gson = new Gson(); } @@ -132,7 +136,7 @@ public PendingImportObject constructPendingObservation() { if (!isTimestampMatched()) { // Update the timestamp DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; - String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timestamp)); + String formattedTimeStampValue = formatter.format(observationService.parseDateTime(timestamp)); update.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); // Add original timestamp to changelog entry @@ -179,7 +183,7 @@ private boolean isTimestampMatched() { if (timestamp == null) { return observation.getObservationTimeStamp() == null; } else { - return !OffsetDateTime.parse(timestamp).equals(observation.getObservationTimeStamp()); + return !observationService.parseDateTime(timestamp).equals(observation.getObservationTimeStamp()); } } // private void validateTimeStampValue(String value, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java index 2c30b0bcb..698cf80de 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java @@ -84,8 +84,9 @@ public static OverwrittenData overwrittenData(boolean canOverwrite, BrAPIObservation observation, UUID userId, Program program, - FieldValidator fieldValidator) { - return new OverwrittenData(canOverwrite, isCommit, unitId, trait, phenoColumnName, timestampColumnName, cellData, timestamp, reason, observation, userId, program, fieldValidator); + FieldValidator fieldValidator, + ObservationService observationService) { + return new OverwrittenData(canOverwrite, isCommit, unitId, trait, phenoColumnName, timestampColumnName, cellData, timestamp, reason, observation, userId, program, fieldValidator, observationService); } public static UnchangedData unchangedData(BrAPIObservation observation, Program program) { @@ -126,7 +127,7 @@ public OverwrittenData overwrittenDataBean(boolean canOverwrite, BrAPIObservation observation, UUID userId, Program program) { - return overwrittenData(canOverwrite, isCommit, unitId, trait, phenoColumnName, timestampColumnName, cellData, timestamp, reason, observation, userId, program, fieldValidator); + return overwrittenData(canOverwrite, isCommit, unitId, trait, phenoColumnName, timestampColumnName, cellData, timestamp, reason, observation, userId, program, fieldValidator, observationService); } @Bean diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index ddd6e9100..188508366 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -243,10 +243,10 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext cell.timestamp = tsColByPheno.get(phenoColumnName).getString(rowNum); tsColumnName = tsColByPheno.get(phenoColumnName).name(); - // If timestamp is not valid, set to midnight + // If timestamp is not valid, add a validation error fieldValidator.validateField(tsColumnName, cell.timestamp, null).ifPresent(err->{ - cell.timestamp += MIDNIGHT; - validationErrors.addError(rowNum, err); + cell.timestamp = null; + validationErrors.addError(rowNum + 2, err); // +2 because of excel header row and 1-based row index }); } @@ -259,7 +259,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext BrAPIObservation observation = gson.fromJson(gson.toJson(observationByObsHash.get(observationHash)), BrAPIObservation.class); // Is there a change to the prior data? - if ((!cellData.isBlank() && !cellData.equals(observation.getValue())) || (cell.timestamp != null && !OffsetDateTime.parse(cell.timestamp).equals(observation.getObservationTimeStamp()))) { + if ((!cellData.isBlank() && !cellData.equals(observation.getValue())) || (cell.timestamp != null && !observationService.parseDateTime(cell.timestamp).equals(observation.getObservationTimeStamp()))) { // Is prior data protected? boolean canOverwrite = context.getImportContext().isCommit() && "false".equals( row.getOverwrite() == null ? "false" : row.getOverwrite()); @@ -312,9 +312,6 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext // Validate processed data processedData.getValidationErrors().ifPresent(errList -> errList.forEach(e->validationErrors.addError(rowNum + 2, e))); // +2 to account for header row and excel file 1-based row index - if (validationErrors.hasErrors()) { - throw new ValidatorException(validationErrors); - } // Update import preview statistics and set in the context processedData.updateTally(statistic); @@ -339,6 +336,11 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context.getImportContext().getMappedBrAPIImport().put(rowNum, mappedImportRow); } + // Throw the total list of all validation errors for the import + if (validationErrors.hasErrors()) { + throw new ValidatorException(validationErrors); + } + // Add the pending observation map to the context for use in processing the import context.getAppendOverwriteWorkflowContext().setPendingObservationByHash(pendingObservationByHash); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index d9c808fed..628eca472 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -34,7 +34,9 @@ import javax.inject.Inject; import javax.inject.Singleton; import java.math.BigDecimal; +import java.time.LocalDate; import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.*; @@ -102,6 +104,23 @@ public String getObservationHash(String observationUnitName, String variableName return DigestUtils.sha256Hex(concat); } + public OffsetDateTime parseDateTime(String dateString) { + // Try parsing as ISO-8601 + try { + return OffsetDateTime.parse(dateString); + } catch (DateTimeParseException e) { + // If ISO-8601 parsing fails, try YY-MM-DD format + try { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + LocalDate localDate = LocalDate.parse(dateString, formatter); + return localDate.atStartOfDay().atOffset(ZoneOffset.UTC); + } catch (DateTimeParseException ex) { + // If both parsing attempts fail, return null + return null; + } + } + } + public BrAPIObservation constructNewBrAPIObservation(boolean commit, String germplasmName, String variableName, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java index 594de233c..0af59ba54 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java @@ -55,7 +55,7 @@ public Optional validateField(String fieldName, String value, T // Is this a timestamp field? if (fieldName.startsWith(TIMESTAMP_PREFIX)) { - if (!observationService.validDateValue(value) || !observationService.validDateTimeValue(value)) { + if (!observationService.validDateValue(value) && !observationService.validDateTimeValue(value)) { return Optional.of(new ValidationError(fieldName, dateTimeMessage, HttpStatus.UNPROCESSABLE_ENTITY)); } From cfcdd9ef29fa5312671d153ee2010fe08efc0ca4 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 24 Jun 2024 18:49:59 -0400 Subject: [PATCH 165/203] fix value match check for overwritten data --- .../appendoverwrite/factory/data/OverwrittenData.java | 4 ++-- .../middleware/process/ImportTableProcess.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java index 308658158..98d3fe646 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java @@ -176,14 +176,14 @@ private void createAdditionalInfoChangeLog(BrAPIObservation update) { } private boolean isValueMatched() { - return !cellData.equals(observation.getValue()); + return cellData.equals(observation.getValue()); } private boolean isTimestampMatched() { if (timestamp == null) { return observation.getObservationTimeStamp() == null; } else { - return !observationService.parseDateTime(timestamp).equals(observation.getObservationTimeStamp()); + return observationService.parseDateTime(timestamp).equals(observation.getObservationTimeStamp()); } } // private void validateTimeStampValue(String value, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 188508366..b13c6ea2d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -262,7 +262,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext if ((!cellData.isBlank() && !cellData.equals(observation.getValue())) || (cell.timestamp != null && !observationService.parseDateTime(cell.timestamp).equals(observation.getObservationTimeStamp()))) { // Is prior data protected? - boolean canOverwrite = context.getImportContext().isCommit() && "false".equals( row.getOverwrite() == null ? "false" : row.getOverwrite()); + boolean canOverwrite = "false".equals( row.getOverwrite() == null ? "false" : row.getOverwrite()); // Clone the trait Trait changeTrait = gson.fromJson(gson.toJson(traitByPhenoColName.get(phenoColumnName)), Trait.class); From 311c148a5b8cf5a8fba9d2a7e9495567dc451bf8 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 24 Jun 2024 20:20:52 -0400 Subject: [PATCH 166/203] include appended timestamps in initial data --- .../factory/data/InitialData.java | 18 ++++++++++++++++++ .../factory/data/ProcessedDataFactory.java | 8 ++++++-- .../middleware/process/ImportTableProcess.java | 2 ++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java index b74a03aa1..1b14a12a4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java @@ -35,6 +35,8 @@ import org.breedinginsight.model.User; import org.breedinginsight.utilities.Utilities; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -47,7 +49,9 @@ public class InitialData extends VisitedObservationData { String germplasmName; BrAPIStudy study; String cellData; + String timestamp; String phenoColumnName; + String timestampColumnName; Trait trait; ExperimentObservation row; UUID trialId; @@ -67,7 +71,9 @@ public class InitialData extends VisitedObservationData { String germplasmName, BrAPIStudy study, String cellData, + String timestamp, String phenoColumnName, + String timestampColumnName, Trait trait, ExperimentObservation row, UUID trialId, @@ -85,7 +91,9 @@ public class InitialData extends VisitedObservationData { this.germplasmName = germplasmName; this.study = study; this.cellData = cellData; + this.timestamp = timestamp; this.phenoColumnName = phenoColumnName; + this.timestampColumnName = timestampColumnName; this.trait = trait; this.row = row; this.trialId = trialId; @@ -107,6 +115,9 @@ public Optional> getValidationErrors() { // Validate observation value fieldValidator.validateField(phenoColumnName, cellData, trait).ifPresent(errors::add); + // Validate timestamp + fieldValidator.validateField(timestampColumnName, timestamp, null).ifPresent(errors::add); + return Optional.ofNullable(errors.isEmpty() ? null : errors); } @@ -133,6 +144,13 @@ public PendingImportObject constructPendingObservation() { user, program); + // Add a timestamp if included + if (timestamp != null && !timestamp.isBlank()) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(observationService.parseDateTime(timestamp)); + newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + } + // Construct a pending observation with a status set to NEW return new PendingImportObject<>(ImportObjectState.NEW, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java index 698cf80de..1dbb34a32 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java @@ -56,7 +56,9 @@ public static InitialData initialData(String brapiReferenceSource, String germplasmName, BrAPIStudy study, String cellData, + String timestamp, String phenoColumnName, + String timestampColumnName, Trait trait, ExperimentObservation row, UUID trialId, @@ -69,7 +71,7 @@ public static InitialData initialData(String brapiReferenceSource, FieldValidator fieldValidator, StudyService studyService, ObservationService observationService) { - return new InitialData(brapiReferenceSource, isCommit, germplasmName, study, cellData, phenoColumnName, trait, row, trialId, studyId, unitId, studyYear, observationUnit, user, program, fieldValidator, studyService, observationService); + return new InitialData(brapiReferenceSource, isCommit, germplasmName, study, cellData, timestamp, phenoColumnName, timestampColumnName, trait, row, trialId, studyId, unitId, studyYear, observationUnit, user, program, fieldValidator, studyService, observationService); } public static OverwrittenData overwrittenData(boolean canOverwrite, @@ -100,7 +102,9 @@ public InitialData initialDataBean(String brapiReferenceSource, String germplasmName, BrAPIStudy study, String cellData, + String timestamp, String phenoColumnName, + String timestampColumnName, Trait trait, ExperimentObservation row, UUID trialId, @@ -110,7 +114,7 @@ public InitialData initialDataBean(String brapiReferenceSource, BrAPIObservationUnit observationUnit, User user, Program program) { - return initialData(brapiReferenceSource, isCommit, germplasmName, study, cellData, phenoColumnName, trait, row, trialId, studyId, unitId, studyYear, observationUnit, user, program, fieldValidator, studyService, observationService); + return initialData(brapiReferenceSource, isCommit, germplasmName, study, cellData, timestamp, phenoColumnName, timestampColumnName, trait, row, trialId, studyId, unitId, studyYear, observationUnit, user, program, fieldValidator, studyService, observationService); } @Bean diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index b13c6ea2d..8a15bcc29 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -298,7 +298,9 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context.getAppendOverwriteWorkflowContext().getPendingGermplasmByOUId().get(unitId).getBrAPIObject().getGermplasmName(), context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getBrAPIObject(), cellData, + cell.timestamp, phenoColumnName, + tsColumnName, initialTrait, row, pendingTrial.getId(), From 53269ee8f7c92395b599921e5d890fa365a19186 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 24 Jun 2024 20:45:23 -0400 Subject: [PATCH 167/203] clear statistics before processing import table --- .../factory/data/OverwrittenData.java | 32 ------------------- .../middleware/process/AppendStatistic.java | 11 ++++--- .../process/ImportTableProcess.java | 3 ++ 3 files changed, 10 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java index 98d3fe646..78bd51a30 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java @@ -186,36 +186,4 @@ private boolean isTimestampMatched() { return observationService.parseDateTime(timestamp).equals(observation.getObservationTimeStamp()); } } -// private void validateTimeStampValue(String value, -// String columnHeader, ValidationErrors validationErrors, int row) { -// if (StringUtils.isBlank(value)) { -// log.debug(String.format("skipping validation of observation timestamp because there is no value.\n\tvariable: %s\n\trow: %d", columnHeader, row)); -// return; -// } -// if (!validDateValue(value) && !validDateTimeValue(value)) { -// addRowError(columnHeader, "Incorrect datetime format detected. Expected YYYY-MM-DD or YYYY-MM-DDThh:mm:ss+hh:mm", validationErrors, row); -// } -// -// } - - private boolean validDateValue(String value) { - DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE; - try { - formatter.parse(value); - } catch (DateTimeParseException e) { - return false; - } - return true; - } - - private boolean validDateTimeValue(String value) { - DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; - try { - formatter.parse(value); - } catch (DateTimeParseException e) { - return false; - } - return true; - } - } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java index 9d9c628a6..0893df9f2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java @@ -26,14 +26,18 @@ @Prototype public class AppendStatistic { - private final HashSet environmentNames; - private final HashSet observationUnitIds; - private final HashSet gids; + private HashSet environmentNames; + private HashSet observationUnitIds; + private HashSet gids; private int newCount; private int existingCount; private int mutatedCount; public AppendStatistic() { + this.clearData(); + } + + public void clearData() { this.environmentNames = new HashSet<>(); this.observationUnitIds = new HashSet<>(); this.gids = new HashSet<>(); @@ -41,7 +45,6 @@ public AppendStatistic() { this.existingCount = 0; this.mutatedCount = 0; } - public int incrementNewCount(Integer value) { int increment = 0; if (value == null) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 8a15bcc29..3743cd05e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -206,6 +206,9 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext // Build new pending observation data for each phenotype Map> pendingObservationByHash = new HashMap<>(); + // In case the user aborted an import, clear any old preview statistics before processing the import + statistic.clearData(); + // Build pending import data maps for each row for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { Integer rowNum = i; From c57df5a4e1c11f0966d41cf374ea230cf5af6531 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 24 Jun 2024 23:28:14 -0400 Subject: [PATCH 168/203] use exisiting processor workaround in empty data bean --- .../factory/data/EmptyData.java | 129 ++++++++++++++++++ .../factory/data/ProcessedDataFactory.java | 34 +++++ .../process/ImportTableProcess.java | 33 ++++- 3 files changed, 189 insertions(+), 7 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/EmptyData.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/EmptyData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/EmptyData.java new file mode 100644 index 000000000..6c0df56ad --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/EmptyData.java @@ -0,0 +1,129 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; + +import io.micronaut.context.annotation.Prototype; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.model.User; +import org.breedinginsight.utilities.Utilities; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Prototype +public class EmptyData extends VisitedObservationData { + String brapiReferenceSource; + boolean isCommit; + String germplasmName; + BrAPIStudy study; + String phenoColumnName; + UUID trialId; + UUID studyId; + UUID unitId; + String studyYear; + BrAPIObservationUnit observationUnit; + User user; + Program program; + private final StudyService studyService; + private final ObservationService observationService; + + public EmptyData(String brapiReferenceSource, + boolean isCommit, + String germplasmName, + BrAPIStudy study, + String phenoColumnName, + UUID trialId, + UUID studyId, + UUID unitId, + String studyYear, + BrAPIObservationUnit observationUnit, + User user, + Program program, + StudyService studyService, + ObservationService observationService) { + this.brapiReferenceSource = brapiReferenceSource; + this.isCommit = isCommit; + this.germplasmName = germplasmName; + this.study = study; + this.phenoColumnName = phenoColumnName; + this.trialId = trialId; + this.studyId = studyId; + this.unitId = unitId; + this.studyYear = studyYear; + this.observationUnit = observationUnit; + this.user = user; + this.program = program; + this.studyService = studyService; + this.observationService = observationService; + } + + @Override + public Optional> getValidationErrors() { + return Optional.empty(); + } + + @Override + public PendingImportObject constructPendingObservation() { + /** + * TODO: fix the front end experiment import preview table so that it won't break if a table row has + * an empty observations array when there are phenotype columns. Once this is fixed on the front end, + * delete the work-around below and simply have this method return null. + */ + + String seasonDbId = studyService.yearToSeasonDbIdFromDatabase(studyYear, program.getId()); + + // Generate a new ID for the observation + UUID observationId = UUID.randomUUID(); + + // Construct the new observation + BrAPIObservation newObservation = observationService.constructNewBrAPIObservation(isCommit, + germplasmName, + phenoColumnName, + study, + seasonDbId, + observationUnit, + "", // the value of the observation is empty + trialId, + studyId, + unitId, + observationId, + brapiReferenceSource, + user, + program); + + // Construct a pending observation with a status set to NEW + return new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); + } + + @Override + public void updateTally(AppendStatistic statistic) { + + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java index 1dbb34a32..84f8bcf75 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java @@ -95,6 +95,23 @@ public static UnchangedData unchangedData(BrAPIObservation observation, Program return new UnchangedData(observation, program); } + public static EmptyData emptyData(String brapiReferenceSource, + boolean isCommit, + String germplasmName, + BrAPIStudy study, + String phenoColumnName, + UUID trialId, + UUID studyId, + UUID unitId, + String studyYear, + BrAPIObservationUnit observationUnit, + User user, + Program program, + StudyService studyService, + ObservationService observationService) { + return new EmptyData(brapiReferenceSource, isCommit, germplasmName, study, phenoColumnName, trialId, studyId, unitId, studyYear, observationUnit, user, program, studyService, observationService); + } + @Bean @Prototype public InitialData initialDataBean(String brapiReferenceSource, @@ -139,5 +156,22 @@ public OverwrittenData overwrittenDataBean(boolean canOverwrite, public UnchangedData unchangedDataBean(BrAPIObservation observation, Program program) { return unchangedData(observation, program); } + + @Bean + @Prototype + public EmptyData emptyDataBean(String brapiReferenceSource, + boolean isCommit, + String germplasmName, + BrAPIStudy study, + String phenoColumnName, + UUID trialId, + UUID studyId, + UUID unitId, + String studyYear, + BrAPIObservationUnit observationUnit, + User user, + Program program) { + return emptyData(brapiReferenceSource, isCommit, germplasmName, study, phenoColumnName, trialId, studyId, unitId, studyYear, observationUnit, user, program, studyService, observationService); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 3743cd05e..44b849533 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -247,7 +247,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext tsColumnName = tsColByPheno.get(phenoColumnName).name(); // If timestamp is not valid, add a validation error - fieldValidator.validateField(tsColumnName, cell.timestamp, null).ifPresent(err->{ + fieldValidator.validateField(tsColumnName, cell.timestamp, null).ifPresent(err -> { cell.timestamp = null; validationErrors.addError(rowNum + 2, err); // +2 because of excel header row and 1-based row index }); @@ -255,6 +255,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext } VisitedObservationData processedData = null; + // Is there prior observation data for this unit + var? if (observationByObsHash.containsKey(observationHash)) { @@ -265,7 +266,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext if ((!cellData.isBlank() && !cellData.equals(observation.getValue())) || (cell.timestamp != null && !observationService.parseDateTime(cell.timestamp).equals(observation.getObservationTimeStamp()))) { // Is prior data protected? - boolean canOverwrite = "false".equals( row.getOverwrite() == null ? "false" : row.getOverwrite()); + boolean canOverwrite = "false".equals(row.getOverwrite() == null ? "false" : row.getOverwrite()); // Clone the trait Trait changeTrait = gson.fromJson(gson.toJson(traitByPhenoColName.get(phenoColumnName)), Trait.class); @@ -289,7 +290,8 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext processedData = processedDataFactory.unchangedDataBean(observation, program); } - } else { + // + } else if (!cellData.isBlank()) { // Clone the observation unit and trait BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(row.getObsUnitID()).getBrAPIObject()), BrAPIObservationUnit.class); @@ -313,10 +315,27 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext observationUnit, context.getImportContext().getUser(), context.getImportContext().getProgram()); + } else { + // Clone the observation unit + BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(row.getObsUnitID()).getBrAPIObject()), BrAPIObservationUnit.class); + + processedData = processedDataFactory.emptyDataBean(brapiReferenceSource, + context.getImportContext().isCommit(), + context.getAppendOverwriteWorkflowContext().getPendingGermplasmByOUId().get(unitId).getBrAPIObject().getGermplasmName(), + context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getBrAPIObject(), + phenoColumnName, + pendingTrial.getId(), + context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getId(), + UUID.fromString(unitId), + context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getSeasons().get(0), + observationUnit, + context.getImportContext().getUser(), + context.getImportContext().getProgram() + ); } // Validate processed data - processedData.getValidationErrors().ifPresent(errList -> errList.forEach(e->validationErrors.addError(rowNum + 2, e))); // +2 to account for header row and excel file 1-based row index + processedData.getValidationErrors().ifPresent(errList -> errList.forEach(e -> validationErrors.addError(rowNum + 2, e))); // +2 to account for header row and excel file 1-based row index // Update import preview statistics and set in the context processedData.updateTally(statistic); @@ -328,13 +347,13 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context.getAppendOverwriteWorkflowContext().setStatistic(statistic); // Construct a pending observation - PendingImportObject pendingProcessedData = processedData.constructPendingObservation(); + Optional> pendingProcessedData = Optional.ofNullable(processedData.constructPendingObservation()); // Set the new pending observation in the pending import for the row - mappedImportRow.getObservations().add(pendingProcessedData); + pendingProcessedData.ifPresent(observation -> mappedImportRow.getObservations().add(observation)); // Add pending observation to map - pendingObservationByHash.put(observationHash, pendingProcessedData); + pendingProcessedData.ifPresent(observation -> pendingObservationByHash.put(observationHash, observation)); } // Set the pending import for the row From b3885e77d13803735f1710592b18c26a1f76e3d1 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 24 Jun 2024 23:35:08 -0400 Subject: [PATCH 169/203] add gid to preview statistics --- .../appendoverwrite/middleware/process/ImportTableProcess.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 44b849533..1503ee3fc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -343,7 +343,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext // TODO: change null values to actual data // TODO: change signature to take two args, studyName and unitName statistic.addObservationUnitId(null); - statistic.addGid(null); + statistic.addGid(context.getAppendOverwriteWorkflowContext().getPendingGermplasmByOUId().get(unitId).getBrAPIObject().getAccessionNumber()); context.getAppendOverwriteWorkflowContext().setStatistic(statistic); // Construct a pending observation From 7c72b91ea10e2deea9e8ae0092dc465942c43d85 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 25 Jun 2024 22:55:23 -0400 Subject: [PATCH 170/203] fix canOverwrite boolean flag and observation update exception handling --- .../brapi/v2/dao/BrAPIObservationDAO.java | 2 +- .../processors/ExperimentProcessor.java | 1 + .../factory/action/BrAPIAction.java | 5 +++- .../factory/action/WorkflowCreation.java | 4 +-- .../factory/action/WorkflowUpdate.java | 25 ++++++++----------- .../middleware/commit/BrAPICommit.java | 1 - .../middleware/commit/BrAPIDatasetCommit.java | 14 +++++++++-- .../commit/BrAPIObservationCommit.java | 21 ++++++++++++---- .../commit/BrAPIObservationUnitCommit.java | 5 +++- .../middleware/commit/BrAPIStudyCommit.java | 5 +++- .../middleware/commit/BrAPITrialCommit.java | 14 +++++++++-- .../middleware/commit/LocationCommit.java | 5 +++- .../process/ImportTableProcess.java | 6 ++++- 13 files changed, 76 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java index 6176a9f65..eed10f7fb 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java @@ -189,7 +189,7 @@ public List getObservationsByDbIds(List dbIds, Program } return getProgramObservations(program.getId()).values().stream() - .filter(o -> dbIds.contains(o.getObservationUnitDbId())) + .filter(o -> dbIds.contains(o.getObservationDbId())) .collect(Collectors.toList()); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index f27d89338..c761c358c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -433,6 +433,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program if (observation == null) { throw new Exception("Null observation"); } + BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); if (updatedObs == null) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java index 9648d8379..96bcf845a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java @@ -20,6 +20,9 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import java.util.Optional; @@ -39,7 +42,7 @@ public interface BrAPIAction { * @return An Optional containing the relevant BrAPI state after executing the action. * @throws ApiException if an error occurs during the execution of the action. */ - Optional execute() throws ApiException; + Optional execute() throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException; /** * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java index 05ab4d096..6b1e7cd28 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java @@ -50,7 +50,7 @@ protected WorkflowCreation(ExperimentImportEntity entity) { * @return an Optional containing the BrAPI state after execution * @throws ApiException if an error occurs during execution */ - public Optional execute() throws ApiException { + public Optional execute() throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { List newMembers = entity.copyWorkflowMembers(ImportObjectState.NEW); try { List createdMembers = entity.brapiPost(newMembers); @@ -58,7 +58,7 @@ public Optional execute() throws ApiException { return Optional.of(new BrAPICreationState<>(createdMembers)); } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { log.error("Error creating..."); - throw new InternalServerException("Error creating...", e); + throw e; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java index 61f80d711..ae13ba303 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java @@ -25,6 +25,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity; +import org.checkerframework.checker.units.qual.A; import java.util.List; import java.util.Optional; @@ -63,20 +64,16 @@ public Optional> getBrAPIState() { } } + protected Optional saveAndUpdateCache(List members) throws IllegalArgumentException, ApiException { + + if (members == null) { + throw new IllegalArgumentException("BrAPI entity cannot be null"); + } + List savedMembers = entity.brapiPut(members); + entity.updateWorkflow(savedMembers); + return Optional.of(new BrAPIUpdateState(savedMembers)); + } - protected Optional saveAndUpdateCache(List members) { - return Optional.ofNullable(members).map(changes -> { - try { - List savedMembers = entity.brapiPut(changes); - entity.updateWorkflow(savedMembers); - return new BrAPIUpdateState(savedMembers); - } catch (ApiException e) { - // TODO: add specific error messages to entity service - log.error("Error updating..."); - throw new InternalServerException("Error updating...", e); - } - }); - } @Getter public class BrAPIUpdateState implements BrAPIState { @@ -84,7 +81,7 @@ public class BrAPIUpdateState implements BrAPIState { public BrAPIUpdateState(List existingMembers) { this.members = existingMembers; } - public boolean restore() { + public boolean restore() throws ApiException { return saveAndUpdateCache(this.getMembers()).isPresent(); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java index 84a9f7e63..1b6cbd7ee 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java @@ -36,7 +36,6 @@ public BrAPICommit(BrAPIDatasetCommit brAPIDatasetCommit, BrAPIObservationUnitCommit brAPIObservationUnitCommit, BrAPIObservationCommit brAPIObservationCommit) { - // TODO: add methods to entity/action classes to register and watch for required foreign key values so order does not have to be hard-wired // Note: the order is important because system-generated dbIds from prior steps are used as foreign keys in // subsequent steps this.middleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link( diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index b7f92086f..15c17145c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -28,6 +28,10 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import java.util.Optional; @@ -59,7 +63,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext priorDatasets = datasetUpdate.getBrAPIState().map(d -> d); log.info("adding new observation variables to datasets"); updatedDatasets = datasetUpdate.execute().map(d -> (WorkflowUpdate.BrAPIUpdateState) d); - } catch (ApiException e) { + } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); } @@ -75,7 +79,13 @@ public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareCont createdDatasets.ifPresent(WorkflowCreation.BrAPICreationState::undo); // Revert any changes made to datasets in the BrAPI service - priorDatasets.ifPresent(WorkflowUpdate.BrAPIUpdateState::restore); + priorDatasets.ifPresent(state -> { + try { + state.restore(); + } catch (ApiException e) { + log.error("Error trying to restore BrAPI variable state: " + Utilities.generateApiExceptionLogMessage(e), e); + } + }); // Undo the prior local transaction return compensatePrior(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java index 3010ad678..b0ce7e696 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -28,6 +28,10 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import java.util.Optional; @@ -59,7 +63,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext log.info("updating existing observations in the BrAPI service"); updatedObservations = brAPIObservationUpdate.execute().map(s -> (WorkflowUpdate.BrAPIUpdateState) s); - } catch (ApiException e) { + } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); } @@ -72,11 +76,18 @@ public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareCont // Tag an error if it occurred in this local transaction context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); - // Delete any created observations from the BrAPI service - createdBrAPIObservations.ifPresent(WorkflowCreation.BrAPICreationState::undo); - // Revert any changes made to observations in the BrAPI service - priorBrAPIObservations.ifPresent(WorkflowUpdate.BrAPIUpdateState::restore); + // Delete any created observations from the BrAPI service + createdBrAPIObservations.ifPresent(WorkflowCreation.BrAPICreationState::undo); + + // Revert any changes made to observations in the BrAPI service + priorBrAPIObservations.ifPresent(state -> { + try { + state.restore(); + } catch (ApiException e) { + log.error("Error trying to restore BrAPI variable state: " + Utilities.generateApiExceptionLogMessage(e), e); + } + }); // Undo the prior local transaction return compensatePrior(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index c2b7533d0..257f03d06 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -26,6 +26,9 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import javax.inject.Inject; import java.util.Optional; @@ -47,7 +50,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext brAPIObservationUnitCreation = brAPICreationFactory.observationUnitWorkflowCreationBean(context); log.info("creating new observation units in the BrAPI service"); createdBrAPIObservationUnits = brAPIObservationUnitCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); - } catch (ApiException e) { + } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java index d3f8eae17..8928d5ca6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -26,6 +26,9 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import javax.inject.Inject; import java.util.Optional; @@ -47,7 +50,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext brAPIStudyCreation = brAPICreationFactory.studyWorkflowCreationBean(context); log.info("creating new studies in the BrAPI service"); createdBrAPIStudies = brAPIStudyCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); - } catch (ApiException e) { + } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index 2b4373781..48fafcf37 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -28,6 +28,10 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import java.util.Optional; @@ -59,7 +63,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext log.info("updating existing trials in the BrAPI service"); updatedTrials = brAPITrialUpdate.execute().map(s -> (WorkflowUpdate.BrAPIUpdateState) s); - } catch (ApiException e) { + } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); } @@ -76,7 +80,13 @@ public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareCont createdBrAPITrials.ifPresent(WorkflowCreation.BrAPICreationState::undo); // Revert any changes made to trials in the BrAPI service - priorBrAPITrials.ifPresent(WorkflowUpdate.BrAPIUpdateState::restore); + priorBrAPITrials.ifPresent(state -> { + try { + state.restore(); + } catch (ApiException e) { + log.error("Error trying to restore BrAPI variable state: " + Utilities.generateApiExceptionLogMessage(e), e); + } + }); // Undo the prior local transaction return compensatePrior(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index 7b72a37df..0697a4bbc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -26,6 +26,9 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import javax.inject.Inject; import java.util.Optional; @@ -47,7 +50,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext locationCreation = brAPICreationFactory.locationWorkflowCreationBean(context); log.info("creating new locationss in the Deltabreed database"); createdLocations = locationCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); - } catch (ApiException e) { + } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 1503ee3fc..8b7614dce 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -266,7 +266,11 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext if ((!cellData.isBlank() && !cellData.equals(observation.getValue())) || (cell.timestamp != null && !observationService.parseDateTime(cell.timestamp).equals(observation.getObservationTimeStamp()))) { // Is prior data protected? - boolean canOverwrite = "false".equals(row.getOverwrite() == null ? "false" : row.getOverwrite()); + /** + * For preview purposes all data can be treated as overwritable, but data cannot be + * overwritten if changes are to be committed and the user has not chosen to overwrite + */ + boolean canOverwrite = !context.getImportContext().isCommit() || !"false".equals(row.getOverwrite() == null ? "false" : row.getOverwrite()); // Clone the trait Trait changeTrait = gson.fromJson(gson.toJson(traitByPhenoColName.get(phenoColumnName)), Trait.class); From d52b54d30e6699920a73daf17f304c6950787568 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 28 Jun 2024 13:43:30 -0400 Subject: [PATCH 171/203] remove vestigial reddison config setting --- src/main/resources/application.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index c207e0bee..10acaf1f0 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -182,7 +182,6 @@ email: from: ${EMAIL_FROM} redisson: - shutdown-on-stop: false single-server-config: address: ${REDIS_URL:`redis://localhost:6379`} connect-timeout: ${REDIS_TIMEOUT:30s} From 7fdd49e70bfbbec326e37b3f1f3aa7c0a0f4fa46 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 28 Jun 2024 14:13:59 -0400 Subject: [PATCH 172/203] delete unused create-new-environment workflow --- .../CreateNewEnvironmentWorkflow.java | 55 ------------------- 1 file changed, 55 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java deleted file mode 100644 index 1b7eba9ce..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; - -import lombok.Getter; -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; - -import javax.inject.Singleton; -import java.util.Optional; - -@Getter -@Singleton -public class CreateNewEnvironmentWorkflow implements ExperimentWorkflow { - private final ExperimentWorkflowNavigator.Workflow workflow; - - public CreateNewEnvironmentWorkflow(){ - this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_ENVIRONMENT; - } - - @Override - public Optional process(ImportServiceContext context) { - // Workflow processing the context - ImportWorkflow workflow = ImportWorkflow.builder() - .id(getWorkflow().getId()) - .name(getWorkflow().getName()) - .build(); - - // No-preview result - Optional result = Optional.of(ImportWorkflowResult.builder() - .workflow(workflow) - .importPreviewResponse(Optional.empty()) - .build()); - - // Skip this workflow unless appending a new environment - if (context != null && !this.workflow.isEqual(context.getWorkflow())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return result; - } - - // Start processing the import... - return result; - } - - @Override - public int getOrder() { - return 3; - } - -} From fff2a9590b5164964cd3fef2b42753c26bb7708f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 28 Jun 2024 15:29:12 -0400 Subject: [PATCH 173/203] delete unused append-workflow from enum --- .../processors/experiment/ExperimentWorkflowNavigator.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index fdb68a548..19918c6bd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -77,8 +77,7 @@ public List getWorkflows() { */ public enum Workflow { NEW_OBSERVATION("new-experiment","Create new experiment"), - APPEND_OVERWRITE("append-dataset", "Append experimental dataset"), - APPEND_ENVIRONMENT("append-environment", "Create new experimental environment"); + APPEND_OVERWRITE("append-dataset", "Append experimental dataset"); private String id; private String name; From 4c838a45495391deb1f160b7682db4a35ee30ae9 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 28 Jun 2024 16:32:24 -0400 Subject: [PATCH 174/203] fetch workflow ids for experiment integration tests --- .../AppendOverwritePhenotypesWorkflow.java | 2 +- .../importer/ExperimentFileImportTest.java | 57 +++++++++++++------ 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index cf2e2ff0f..759d91328 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -115,7 +115,7 @@ public Optional process(ImportServiceContext context) { return result; } - // BUild and return the preview response + // Build and return the preview response ImportPreviewResponse response = new ImportPreviewResponse(); response.setStatistics(processedPreviewContext.getAppendOverwriteWorkflowContext().getStatistic().constructPreviewMap()); response.setRows(new ArrayList<>(processedPreviewContext.getImportContext().getMappedBrAPIImport().values())); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 21cde0fa6..19e84b9af 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -24,6 +24,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.client.RxHttpClient; import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.netty.cookies.NettyCookie; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; import lombok.SneakyThrows; @@ -77,6 +78,7 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import static io.micronaut.http.HttpRequest.GET; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.*; @@ -85,10 +87,6 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class ExperimentFileImportTest extends BrAPITest { - private static final String OVERWRITE = "overwrite"; - private static final String APPEND_OVERWRITE_WORKFLOW = "append-dataset"; - private static final String NEW_EXPERIMENT_WORKFLOW = "new-experiment"; - private FannyPack securityFp; private String mappingId; private BiUserEntity testUser; @@ -148,6 +146,8 @@ public class ExperimentFileImportTest extends BrAPITest { (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .registerTypeAdapter(BrAPIPagination.class, new PaginationTypeAdapter()) .create(); + private String newExperimentWorkflowId; + private String appendOverwriteWorkflowId; @BeforeAll public void setup() { @@ -157,8 +157,33 @@ public void setup() { testUser = (BiUserEntity) setupObjects.get("testUser"); securityFp = (FannyPack) setupObjects.get("securityFp"); + /** + * Implicit test that the workflow ids are assigned in the following order + */ + newExperimentWorkflowId = getNewExperimentWorkflowId(0); + appendOverwriteWorkflowId = getNewExperimentWorkflowId(1); + } + /** + * TODO: assumes new workflow is first in list, doesn't look at position property, would be more robust to + * look at that instead of assuming order + * @return + */ + public String getNewExperimentWorkflowId(int workflowIndex) { + + // GET /import/mappings{?importName} + Flowable> call = client.exchange( + GET("/import/mappings/"+mappingId+"/workflows").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + + return JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(workflowIndex).getAsJsonObject().get("id").getAsString(); + } /* Tests - new experiment @@ -661,7 +686,7 @@ public void importNewObsVarExistingOu() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, NEW_EXPERIMENT_WORKFLOW); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -689,7 +714,7 @@ public void importNewObsVarExistingOu() { newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId, APPEND_OVERWRITE_WORKFLOW); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -725,7 +750,7 @@ public void importNewObsVarByObsUnitId() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, NEW_EXPERIMENT_WORKFLOW); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -740,7 +765,7 @@ public void importNewObsVarByObsUnitId() { newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId, APPEND_OVERWRITE_WORKFLOW); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -777,7 +802,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); // empty dataset - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, NEW_EXPERIMENT_WORKFLOW); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -805,7 +830,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, commit, client, program, mappingId, APPEND_OVERWRITE_WORKFLOW); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -846,7 +871,7 @@ public void importNewObsExistingOu(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, NEW_EXPERIMENT_WORKFLOW); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -874,7 +899,7 @@ public void importNewObsExistingOu(boolean commit) { newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, APPEND_OVERWRITE_WORKFLOW); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -1040,7 +1065,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, NEW_EXPERIMENT_WORKFLOW); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1069,7 +1094,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, APPEND_OVERWRITE_WORKFLOW); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -1114,7 +1139,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, NEW_EXPERIMENT_WORKFLOW); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1145,7 +1170,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newObservation.put(traits.get(0).getObservationVariableName(), ""); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, APPEND_OVERWRITE_WORKFLOW); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); From 82a0e5d955f0b55b1b28bd9dbce1645d289214b0 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 28 Jun 2024 20:02:15 -0400 Subject: [PATCH 175/203] fix new-exp test and remove processor from import service layer --- .../model/imports/DomainImportService.java | 23 +-- .../ExperimentImportService.java | 7 +- .../importer/ExperimentFileImportTest.java | 171 ++++++++++++------ .../brapps/importer/ImportTestUtils.java | 19 ++ 4 files changed, 138 insertions(+), 82 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index ef27af1d6..a8dce4090 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -34,19 +34,11 @@ @Singleton @Slf4j public abstract class DomainImportService implements BrAPIImportService { - - // TODO: delete processor fields once WorkflowNavigator is used - private final Provider experimentProcessorProvider; - private final Provider processorManagerProvider; private final Workflow workflowNavigator; - public DomainImportService(Provider experimentProcessorProvider, - Provider processorManagerProvider, - Workflow workflowNavigator) + public DomainImportService(Workflow workflowNavigator) { - this.experimentProcessorProvider = experimentProcessorProvider; - this.processorManagerProvider = processorManagerProvider; this.workflowNavigator = workflowNavigator; } @@ -67,7 +59,7 @@ public ImportPreviewResponse process(ImportServiceContext context) .filter(workflow -> !workflow.isEmpty()) .ifPresent(workflow -> log.info("Workflow: " + workflow)); - if (ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE.getId().equals(context.getWorkflow()) || ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION.getId().equals(context.getWorkflow())) { + Optional result = workflowNavigator.process(context); // Throw any exceptions caught during workflow processing @@ -75,18 +67,7 @@ public ImportPreviewResponse process(ImportServiceContext context) throw result.flatMap(ImportWorkflowResult::getCaughtException).get(); } - return result.flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); - } else { - // TODO: remove this case and just use workflow navigator once all integration tests have been updated to use workflow ids - return processorManagerProvider.get().process(context.getBrAPIImports(), - List.of(experimentProcessorProvider.get()), - context.getData(), - context.getProgram(), - context.getUpload(), - context.getUser(), - context.isCommit()); - } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 793506c73..9e27d0e20 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -39,13 +39,10 @@ public class ExperimentImportService extends DomainImportService { private final String IMPORT_TYPE_ID = "ExperimentImport"; - // TODO: delete processor fields once WorkflowNavigator is used @Inject - public ExperimentImportService(Provider experimentProcessorProvider, - Provider processorManagerProvider, - ExperimentWorkflowNavigator workflowNavigator) + public ExperimentImportService(ExperimentWorkflowNavigator workflowNavigator) { - super(experimentProcessorProvider, processorManagerProvider, workflowNavigator); + super(workflowNavigator); } @Override diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 19e84b9af..86d7a27e9 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -160,8 +160,8 @@ public void setup() { /** * Implicit test that the workflow ids are assigned in the following order */ - newExperimentWorkflowId = getNewExperimentWorkflowId(0); - appendOverwriteWorkflowId = getNewExperimentWorkflowId(1); + newExperimentWorkflowId = getExperimentWorkflowId(0); + appendOverwriteWorkflowId = getExperimentWorkflowId(1); } @@ -170,7 +170,7 @@ public void setup() { * look at that instead of assuming order * @return */ - public String getNewExperimentWorkflowId(int workflowIndex) { + public String getExperimentWorkflowId(int workflowIndex) { // GET /import/mappings{?importName} Flowable> call = client.exchange( @@ -221,16 +221,20 @@ public void importNewExpNewLocNoObsSuccess() { validRow.put(Columns.COLUMN, "1"); validRow.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String workflowId = "new-experiment"; + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId, newExperimentWorkflowId); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + // TODO: remove this + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); @@ -279,16 +283,18 @@ public void importNewExpMultiNewEnvSuccess() { secondEnv.put(Columns.COLUMN, "1"); secondEnv.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(2, previewRows.size()); JsonObject firstRow = previewRows.get(0).getAsJsonObject(); @@ -326,7 +332,8 @@ public void importExistingExpAndEnvErrorMessage() { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + JsonObject expResult = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + //JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); Map dupExp = new HashMap<>(); dupExp.put(Columns.GERMPLASM_GID, "1"); @@ -343,16 +350,17 @@ public void importExistingExpAndEnvErrorMessage() { dupExp.put(Columns.ROW, "1"); dupExp.put(Columns.COLUMN, "1"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, false, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + expResult = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, false, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, expResult.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + expResult); + assertTrue(expResult.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); } @Test @@ -375,9 +383,10 @@ public void importNewEnvNoObsSuccess() { newEnv.put(Columns.ROW, "1"); newEnv.put(Columns.COLUMN, "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); @@ -409,43 +418,53 @@ public void verifyMissingDataThrowsError(boolean commit) { Map noGID = new HashMap<>(base); noGID.remove(Columns.GERMPLASM_GID); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit, newExperimentWorkflowId); Map noExpTitle = new HashMap<>(base); noExpTitle.remove(Columns.EXP_TITLE); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit, newExperimentWorkflowId); Map noExpUnit = new HashMap<>(base); noExpUnit.remove(Columns.EXP_UNIT); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit, newExperimentWorkflowId); Map noExpType = new HashMap<>(base); noExpType.remove(Columns.EXP_TYPE); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit, newExperimentWorkflowId); Map noEnv = new HashMap<>(base); noEnv.remove(Columns.ENV); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit, newExperimentWorkflowId); Map noEnvLoc = new HashMap<>(base); noEnvLoc.remove(Columns.ENV_LOCATION); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); Map noExpUnitId = new HashMap<>(base); noExpUnitId.remove(Columns.EXP_UNIT_ID); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit, newExperimentWorkflowId); Map noExpRep = new HashMap<>(base); noExpRep.remove(Columns.REP_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit, newExperimentWorkflowId); Map noExpBlock = new HashMap<>(base); noExpBlock.remove(Columns.BLOCK_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit, newExperimentWorkflowId); Map noEnvYear = new HashMap<>(base); noEnvYear.remove(Columns.ENV_YEAR); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); } @Test @@ -469,7 +488,8 @@ public void importNewExpWithObsVar() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -519,7 +539,9 @@ public void verifyDiffYearSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); + } @ParameterizedTest @@ -557,7 +579,8 @@ public void verifyDiffLocSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); } @ParameterizedTest @@ -582,7 +605,8 @@ public void importNewExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -622,7 +646,9 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "Red"); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); + } @ParameterizedTest @@ -645,21 +671,24 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + //importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); Map newOU = new HashMap<>(newExp); newOU.put(Columns.EXP_UNIT_ID, "a-2"); newOU.put(Columns.ROW, "1"); newOU.put(Columns.COLUMN, "2"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, true, client, program, mappingId, newExperimentWorkflowId); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); @@ -938,7 +967,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -966,7 +995,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "2"); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); } /* @@ -975,6 +1004,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { - a new experiment is created after the first experiment - verify the second experiment gets created successfully */ + //TODO: this one @Test @SneakyThrows public void importSecondExpAfterFirstExpWithObs() { @@ -996,7 +1026,8 @@ public void importSecondExpAfterFirstExpWithObs() { newExpA.put(Columns.COLUMN, "1"); newExpA.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + //JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + JsonObject resultA = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsA = resultA.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsA.size()); @@ -1024,7 +1055,8 @@ public void importSecondExpAfterFirstExpWithObs() { newExpB.put(Columns.COLUMN, "1"); newExpB.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + //JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + JsonObject resultB = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsB = resultB.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsB.size()); @@ -1554,6 +1586,33 @@ private JsonObject uploadAndVerifyFailure(Program program, File file, String exp JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); + assertEquals(1, rowErrors.size()); + JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); + assertEquals(1, fieldErrors.size()); + JsonObject error = fieldErrors.get(0).getAsJsonObject(); + assertEquals(expectedColumnError, error.get("field").getAsString()); + assertEquals(422, error.get("httpStatusCode").getAsInt()); + + return result; + } + + private JsonObject uploadAndVerifyWorkflowFailure(Program program, File file, String expectedColumnError, boolean commit, String workflowId) throws InterruptedException, IOException { + + //Flowable> call = importTestUtils.uploadDataFile(file, null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(file, null, true, client, program, mappingId, newExperimentWorkflowId); + //JsonObject result = JsonParser.parseString(upload).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); assertEquals(1, rowErrors.size()); JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index 2229b609b..84da9b536 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -128,6 +128,7 @@ public Flowable> uploadWorkflowDataFile(File file, return processCall; } + public HttpResponse getUploadedFile(String importId, RxHttpClient client, Program program, String mappingId) throws InterruptedException { Flowable> call = client.exchange( GET(String.format("/programs/%s/import/mappings/%s/data/%s?mapping=true", program.getId(), mappingId, importId)) @@ -220,6 +221,24 @@ public JsonObject uploadAndFetchWorkflow(File file, return result; } + public JsonObject uploadAndFetchWorkflowNoStatusCheck(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { + Flowable> call = uploadWorkflowDataFile(file, userData, commit, client, program, mappingId, workflowId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + return result; + } + public List createTraits(int numToCreate) { List traits = new ArrayList<>(); for (int i = 0; i < numToCreate; i++) { From d61d58f450e53e74c2825bfefe88246ffae298ac Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 1 Jul 2024 18:08:19 -0400 Subject: [PATCH 176/203] add documentation comments --- .../brapi/v2/dao/BrAPIObservationDAO.java | 11 + .../importer/model/workflow/Workflow.java | 18 ++ .../importer/services/FileImportService.java | 14 +- .../experiment/ExperimentUtilities.java | 253 +++++++++++++++++- .../ExperimentWorkflowNavigator.java | 72 ++++- .../AppendOverwritePhenotypesWorkflow.java | 15 +- .../factory/action/BrAPICreationFactory.java | 88 ++++++ .../validator/field/DateValidator.java | 28 +- .../validator/field/FieldValidator.java | 20 ++ .../validator/field/NominalValidator.java | 28 +- .../validator/field/NumericalValidator.java | 46 +++- .../validator/field/OrdinalValidator.java | 29 +- 12 files changed, 565 insertions(+), 57 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java index eed10f7fb..fea32b569 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java @@ -183,11 +183,22 @@ public List getObservationsByStudyName(List studyNames ); } + /** + * Retrieves a list of observations based on their database IDs and a specific program. + * + * @param dbIds A list of database IDs representing the observations to retrieve. + * @param program The Program object for which the observations belong. + * @return A List of BrAPIObservation objects filtered by the provided database IDs. + * @throws ApiException if an error occurs during the retrieval process. + */ public List getObservationsByDbIds(List dbIds, Program program) throws ApiException { + // Check if the dbIds list is empty and return an empty list if so if(dbIds.isEmpty()) { return Collections.emptyList(); } + // Filter the observations based on the provided program ID and the provided list of dbIds + // Collect the filtered observations into a List and return the result return getProgramObservations(program.getId()).values().stream() .filter(o -> dbIds.contains(o.getObservationDbId())) .collect(Collectors.toList()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java index 45af60609..b82bbd690 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -24,9 +24,27 @@ import java.util.List; import java.util.Optional; +/** + * This Functional Interface represents a Workflow that can be executed as part of an import process. + * It extends the Ordered interface to allow workflows to be ordered in a sequence. + */ @FunctionalInterface public interface Workflow extends Ordered { + + /** + * Process method that defines the logic to be executed as part of the workflow. + * + * @param context the ImportServiceContext object containing necessary information for the workflow + * @return an Optional of ImportWorkflowResult representing the result of the workflow execution + */ Optional process(ImportServiceContext context); + + /** + * Default method to get a list of workflows. + * This method provides a default implementation returning an empty list. + * + * @return a List of ImportWorkflow containing workflows + */ default List getWorkflows() { // Default implementation for getWorkflows method return new ArrayList<>(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 6a1914ab9..fc660d6da 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -571,13 +571,23 @@ public List getSystemMappingByName(String name) { return importMappings; } + /** + * Retrieves the list of import workflows associated with a specific system mapping. + * + * @param mappingId The ID of the system mapping for which to retrieve workflows + * @return A list of ImportWorkflow objects representing the workflows for the specified system mapping + * @throws DoesNotExistException If the system mapping with the given ID does not exist + */ public List getWorkflowsForSystemMapping(UUID mappingId) throws DoesNotExistException { + // Retrieve the import mapping configuration based on the provided mapping ID ImportMapping mappingConfig = importMappingDAO.getMapping(mappingId) .orElseThrow(() -> new DoesNotExistException("Cannot find mapping config associated with upload.")); + + // Get the import service associated with the import type ID from the configuration manager BrAPIImportService importService = configManager.getImportServiceById(mappingConfig.getImportTypeId()) .orElseThrow(() -> new DoesNotExistException("Config with that id does not exist")); + + // Retrieve and return the list of import workflows for the specified system mapping return importService.getWorkflows(); } - - } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 06f241b75..6ef89f326 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -41,6 +41,8 @@ import java.util.*; import java.util.stream.Collectors; + + @Singleton public class ExperimentUtilities { @@ -52,18 +54,47 @@ public class ExperimentUtilities { public ExperimentUtilities() { this.gson = new Gson(); } + + /** + * Checks if the provided list contains any invalid members for the specified class. + * + * @param list The list to be checked for invalid members + * @param clazz The class to check for instance validity + * @return true if the list is null, empty, or contains any member that is not an instance of the specified class; false otherwise + */ public boolean isInvalidMemberListForClass(List list, Class clazz) { - // Check if the input list is of type + // Check if the input list is null, empty, or contains any member that is not an instance of the specified class return list == null || list.isEmpty() || !list.stream().allMatch(clazz::isInstance); } + /** + * This method creates a deep copy of an object using Gson library in Java 8. + * It takes an object of type T and its corresponding class to clone. + * @param obj the object to clone + * @param clazz the class of the object to clone + * @return an Optional containing the cloned object if successful, otherwise an empty Optional + * @throws JsonSyntaxException if there is an issue with JSON syntax during the cloning process + */ public Optional clone(T obj, Class clazz) { try { + // Convert the object to JSON string and then parse it back to the specified class return Optional.ofNullable(gson.fromJson(gson.toJson(obj), clazz)); } catch (JsonSyntaxException e) { + // Return an empty Optional if there is a JsonSyntaxException return Optional.empty(); } } + + /** + * Retrieves a list of new objects of type T from the provided map of pending import objects by filtering out null previews and objects with state other than NEW, + * mapping the BrAPI object from each preview, cloning it to the specified class type, and collecting the non-empty results into a list. + * + * @param objectsByName a map of pending import objects with V keys and PendingImportObject values + * @param clazz the target class type for cloning the BrAPI object + * @param the type of new objects to be retrieved + * @param the type of keys in the map of pending import objects + * @return a list of cloned new objects of type T extracted from the input map + */ public List getNewObjects(Map> objectsByName, Class clazz) { return objectsByName.values().stream() .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW) @@ -73,31 +104,67 @@ public List getNewObjects(Map> objectsByName .map(Optional::get) .collect(Collectors.toList()); } + + /** + * Copies mutated objects from a cache map to a new list. + * Only objects with ImportObjectState MUTATED are included in the copied list. + * + * @param pendingCacheMap a map containing PendingImportObject objects to be copied + * @param clazz a Class object representing the type of objects to be copied + * @return a List of copied objects of type T + */ public List copyMutationsFromCache(Map> pendingCacheMap, Class clazz) { return pendingCacheMap.values().stream() .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED) .map(PendingImportObject::getBrAPIObject) - .map(b->clone(b, clazz)) + .map(b -> clone(b, clazz)) .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors.toList()); } + + /** + * Copies the pending BrAPI objects from the workflow cache map based on the provided import object status. + * + * @param pendingCacheMap a map containing pending import objects with generic values V and T + * @param clazz the class type of the BrAPI object to be cloned + * @param status the import object state to filter the pending objects + * @return a list of cloned BrAPI objects that are in the specified import object state + */ public List copyWorkflowCachePendingBrAPIObjects(Map> pendingCacheMap, Class clazz, ImportObjectState status) { + // Filter the pending import objects by checking if the object is not null and has the specified status return pendingCacheMap.values().stream() .filter(preview -> preview != null && preview.getState() == status) + // Map each pending import object to its corresponding BrAPI object .map(PendingImportObject::getBrAPIObject) - .map(b->clone(b, clazz)) + // Clone the BrAPI object with the provided class type + .map(brApiObject -> clone(brApiObject, clazz)) + // Filter out any empty optionals .filter(Optional::isPresent) + // Unwrap the optional value .map(Optional::get) + // Collect the cloned BrAPI objects into a list .collect(Collectors.toList()); } + + /** + * Retrieves mutations by object ID from a Map of PendingImportObject, filtering based on the object state and applying a DB ID filter. + * + * @param objectsByName A Map with values of type PendingImportObject, used for retrieving the mutations. + * @param dbIdFilter A Function that filters the objects based on their DB ID. + * @param clazz The Class type for the objects in the Map. + * @param Type parameter for the objects in the Map. + * @param Type parameter for the keys in the Map. + * @return A Map of String keys (DB IDs) and objects of type T as values based on the filter logic. + * @throws RuntimeException if an exception occurs while applying the DB ID filter to an object. + */ public Map getMutationsByObjectId(Map> objectsByName, Function dbIdFilter, Class clazz) { return objectsByName.values().stream() .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED) .map(PendingImportObject::getBrAPIObject) - .map(b->clone(b, clazz)) + .map(b -> clone(b, clazz)) .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors @@ -110,27 +177,56 @@ public Map getMutationsByObjectId(Map brapiObj)); } + + /** + * Converts a list of BrAPI imports to a list of ExperimentObservations. + * + * This function takes a list of BrAPIImport objects representing trial imports and converts them + * to ExperimentObservation objects. It utilizes Java 8 stream API to map each BrAPIImport object + * to an ExperimentObservation object and collects the results into a new list. + * + * @param importRows a list of BrAPIImport objects representing trial imports to be converted + * @return a list of ExperimentObservation objects containing the converted data + */ public static List importRowsToExperimentObservations(List importRows) { return importRows.stream() .map(trialImport -> (ExperimentObservation) trialImport) .collect(Collectors.toList()); } + /** + * This method generates a unique key for an observation unit based on the environment and experimental unit ID. + * + * @param importRow the ExperimentObservation object containing the environment and experimental unit ID + * @return a String representing the unique key for the observation unit + */ public static String createObservationUnitKey(ExperimentObservation importRow) { + // Extract the environment and experimental unit ID from the ExperimentObservation object + // and pass them to the createObservationUnitKey method return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); } + /** + * Create Observation Unit Key + * + * This method takes in the name of a study and the name of an observation unit and concatenates them to create a unique key. + * + * @param studyName The name of the study + * @param obsUnitName The name of the observation unit + * @return A string representing the unique key formed by concatenating the study name and observation unit name + */ public static String createObservationUnitKey(String studyName, String obsUnitName) { + // Concatenate the study name and observation unit name to create the unique key return studyName + obsUnitName; } /** - * Returns the single value from the given map, throwing an UnprocessableEntityException if the map does not contain exactly one entry. + * Returns the single value from the provided map if it contains exactly one entry. * - * @param map The map from which to retrieve the single value. - * @param message The error message to include in the UnprocessableEntityException if the map does not contain exactly one entry. - * @return The single value from the map. - * @throws UnprocessableEntityException if the map does not contain exactly one entry. + * @param map the map from which to extract the single value + * @param message the message to be included in the exception thrown if the map does not contain exactly one entry + * @return the single value from the map + * @throws UnprocessableEntityException if the map does not contain only one entry */ public V getSingleEntryValue(Map map, String message) throws UnprocessableEntityException { if (map.size() != 1) { @@ -139,6 +235,38 @@ public V getSingleEntryValue(Map map, String message) throws Unproc return map.values().iterator().next(); } +// Module/Script-level documentation + /** + * This method is used to retrieve the value from a map if the map contains only one entry. + * This method is particularly useful when dealing with scenarios where a single entry is expected. + * It throws an UnprocessableEntityException if the map does not contain exactly one entry, providing a custom error message for clarity. + * Usage: + * Map exampleMap = new HashMap<>(); + * exampleMap.put("exampleKey", 5); + * Integer value = getSingleEntryValue(exampleMap, "Map should contain exactly one entry."); + */ + + /** + * Retrieves the single value from a given map, if the map contains exactly one key-value pair. + * + * @param map The map from which to retrieve the single value. + * @return An Optional containing the single value if the map contains only one key-value pair, + * or an empty Optional if the map is empty or contains more than one entry. + * + * Input: + * - map: The map from which to retrieve the single value. + * + * Output: + * - An Optional containing the single value if the map contains exactly one key-value pair, + * or an empty Optional if the map is empty or contains more than one entry. + * + * Side Effects: + * - None + * + * Usage: + * if you have a map and you want to retrieve the single value associated with a key, you can use + * this function to ensure that the map contains only one entry before retrieving the value. + */ public static Optional getSingleEntryValue(Map map) { Optional value = Optional.empty(); if (map.size() == 1) { @@ -147,6 +275,31 @@ public static Optional getSingleEntryValue(Map map) { return value; } + /* + * Add a given year to the additionalInfo field of the BrAPIStudy, if it does not already exist. + * + * @param program the program to which the study belongs + * @param study the BrAPIStudy object to which the year should be added + * @param year the year to be added to the additionalInfo field + * + * This method checks if the additionalInfo field of the BrAPIStudy object is null, and if so, initializes it with a new JsonObject. + * Then, it checks if the ENV_YEAR key already exists in the additionalInfo object, and if not, adds the given year with the key ENV_YEAR. + * + * @return void + */ + + /* Module Description + * This module contains a method that adds a given year to the additionalInfo field of a BrAPIStudy object within a program's context. + * The purpose of this method is to provide a convenient way to store and retrieve additional information related to the study. + * + * Usage: + * To add a year to the additionalInfo field of a BrAPIStudy object, call this method passing the program, study, and year as parameters. + */ + + /* Side effects: + * This method mutates the state of the BrAPIStudy object by adding the given year to its additionalInfo field. + */ + /* * this will add the given year to the additionalInfo field of the BrAPIStudy (if it does not already exist) * */ @@ -161,70 +314,124 @@ public void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study, Stri } } + /** + * This method is responsible for collating unique ObsUnit IDs from the provided context data. + * + * @param context the AppendOverwriteMiddlewareContext containing the import rows to process + * @return a Set of unique ObsUnit IDs collated from the import rows + * @throws IllegalStateException if any ObsUnit ID is repeated in the import rows + * @throws HttpStatusException if there is a mix of ObsUnit IDs for some but not all rows + */ public static Set collateReferenceOUIds(AppendOverwriteMiddlewareContext context) { + // Initialize variables to track the presence of ObsUnit IDs Set referenceOUIds = new HashSet<>(); boolean hasNoReferenceUnitIds = true; boolean hasAllReferenceUnitIds = true; + + // Iterate through the import rows to process ObsUnit IDs for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); // Check if ObsUnitID is blank if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { + // Set flag to indicate missing ObsUnit ID for current row hasAllReferenceUnitIds = false; } else if (referenceOUIds.contains(importRow.getObsUnitID())) { - // Throw exception if ObsUnitID is repeated throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); } else { // Add ObsUnitID to referenceOUIds referenceOUIds.add(importRow.getObsUnitID()); + // Set flag to indicate presence of ObsUnit ID hasNoReferenceUnitIds = false; } } - if (!hasNoReferenceUnitIds && !hasAllReferenceUnitIds) { - // can't proceed if the import has a mix of ObsUnitId for some but not all rows + if (!hasNoReferenceUnitIds && !hasAllReferenceUnitIds) { + // Throw exception if there is a mix of ObsUnit IDs for some but not all rows throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, ExpImportProcessConstants.ErrMessage.MISSING_OBS_UNIT_ID_ERROR); } + return referenceOUIds; } + /** + * This method sorts a list of items based on a list of sorted fields in ascending order using Java 8 functionality. + * + * @param sortedFields a list of strings representing the fields to sort by + * @param unsortedItems a list of items of generic type T to be sorted + * @param fieldGetter a Function object that extracts the string field from an item of type T + * @return a sorted list of items of type T based on the specified fields + * @throws RuntimeException if there are any exceptions encountered during sorting + */ public List sortByField(List sortedFields, List unsortedItems, Function fieldGetter) { + // Create a case-insensitive map to store the sort order of fields CaseInsensitiveMap sortOrder = new CaseInsensitiveMap<>(); + + // Populate the sortOrder map with the fields and their respective indices from the sortedFields list for (int i = 0; i < sortedFields.size(); i++) { sortOrder.put(sortedFields.get(i), i); } + // Sort the unsortedItems list using a lambda expression to compare items based on the order of specified fields unsortedItems.sort((i1, i2) -> { try { + // Extract the field values of the items using the fieldGetter function String field1 = fieldGetter.apply(i1); String field2 = fieldGetter.apply(i2); + + // Compare the indices of the fields in sortOrder and return the result return Integer.compare(sortOrder.get(field1), sortOrder.get(field2)); } catch (Exception e) { + // Throw a runtime exception if any error occurs during sorting throw new RuntimeException(e); } }); + // Return the sorted list of items return unsortedItems; } + /** + * Constructs a list of BrAPIExternalReference objects for various entities using the given parameters. + * + * @param program the program entity for which external references are to be constructed + * @param referenceSourceBaseName the base name for the reference source + * @param trialId the UUID of the trial entity + * @param datasetId the UUID of the dataset entity + * @param studyId the UUID of the study entity + * @param obsUnitId the UUID of the observation unit entity + * @param observationId the UUID of the observation entity + * @return a list of BrAPIExternalReference objects representing the external references + */ public List constructBrAPIExternalReferences( Program program, String referenceSourceBaseName, UUID trialId, UUID datasetId, UUID studyId, UUID obsUnitId, UUID observationId) { List refs = new ArrayList<>(); + // Add reference for the program entity addReference(refs, program.getId(), referenceSourceBaseName, ExternalReferenceSource.PROGRAMS); + + // Add reference for the trial entity if available if (trialId != null) { addReference(refs, trialId, referenceSourceBaseName, ExternalReferenceSource.TRIALS); } + + // Add reference for the dataset entity if available if (datasetId != null) { addReference(refs, datasetId, referenceSourceBaseName, ExternalReferenceSource.DATASET); } + + // Add reference for the study entity if available if (studyId != null) { addReference(refs, studyId, referenceSourceBaseName, ExternalReferenceSource.STUDIES); } + + // Add reference for the observation unit entity if available if (obsUnitId != null) { addReference(refs, obsUnitId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATION_UNITS); } + + // Add reference for the observation entity if available if (observationId != null) { addReference(refs, observationId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATIONS); } @@ -232,10 +439,32 @@ public List constructBrAPIExternalReferences( return refs; } + /** + * Adds a new reference to the given list of BrAPIExternalReference objects. + * + * @param refs the list of BrAPIExternalReference objects to which the new reference will be added + * @param uuid the UUID to set as the reference ID for the new BrAPIExternalReference + * @param referenceBaseNameSource the base name for the reference source + * @param refSourceName the source of the external reference + */ private void addReference(List refs, UUID uuid, String referenceBaseNameSource, ExternalReferenceSource refSourceName) { + // Create a new BrAPIExternalReference object BrAPIExternalReference reference = new BrAPIExternalReference(); + + // Set the reference source as a combination of reference base name source and external reference source reference.setReferenceSource(String.format("%s/%s", referenceBaseNameSource, refSourceName.getName())); + + // Set the reference ID as the UUID converted to a string reference.setReferenceID(uuid.toString()); + + // Add the new reference to the list of references refs.add(reference); } + /** + * Module overview: This module provides a method to add a new reference to a list of BrAPIExternalReference objects. + * The method takes in the list of references, a UUID, reference base name source, and external reference source. + * It creates a new BrAPIExternalReference object, sets the reference source and ID, and adds it to the list of references. + * This function is useful when dealing with external references in BrAPI-related operations. + * Usage: Call this method with the required parameters to add a new reference to the existing list of references. + */ } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index 19918c6bd..520fc53d5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -42,6 +42,15 @@ public ExperimentWorkflowNavigator(List workflows) { this.workflows = workflows; } + /** + * Process the import service context by executing a series of workflows in order + * + * This method iterates over the list of workflows provided, executing each workflow's process method + * with the given import service context. It then filters out empty results and returns the first non-empty result. + * + * @param context The import service context containing the data to be processed + * @return An Optional containing the first non-empty ImportWorkflowResult from the executed workflows, or an empty Optional if no non-empty result is found + */ @Override public Optional process(ImportServiceContext context) { /** @@ -53,46 +62,81 @@ public Optional process(ImportServiceContext context) { .map(Optional::get) .findFirst(); } + + /** + * Retrieves a list of ImportWorkflow objects containing metadata about each workflow that processed the import context. + * + * @return List of ImportWorkflow objects with workflow metadata + */ public List getWorkflows() { - /** Each workflow returns in the field workflow the metadata about the workflow that processed the import context. - * Loop over all workflows, processing a null context, to collect just the metadata for each workflow - */ List workflowSummaryList = workflows.stream() - .map(workflow->workflow.process(null)) - .filter(Optional::isPresent) - .map(Optional::get) - .map(result->result.getWorkflow()) - .collect(Collectors.toList()); + .map(workflow -> workflow.process(null)) // Process each workflow with a null context + .filter(Optional::isPresent) // Filter out any workflows that do not return a result + .map(Optional::get) // Extract the result from Optional + .map(result -> result.getWorkflow()) // Retrieve the workflow metadata + .collect(Collectors.toList()); // Collect the workflow metadata into a list - // The order field for each workflow is set to the order in the navigator list + // Set the order field for each workflow based on its position in the list for (int i = 0; i < workflowSummaryList.size(); i++) { - workflowSummaryList.get(i).setOrder(i); + workflowSummaryList.get(i).setOrder(i); // Set the order for each workflow } - return workflowSummaryList; + return workflowSummaryList; // Return the list of workflow metadata } /** - * Possible choices of metadata that an experiment workflow instance can use to identify itself + * The Workflow enum represents different workflow types that can be associated with an experiment. */ public enum Workflow { + + /** + * Represents a new observation workflow where a new experiment is created. + * ID: "new-experiment" + * Name: "Create new experiment" + */ NEW_OBSERVATION("new-experiment","Create new experiment"), + + /** + * Represents an append or overwrite workflow where experimental dataset is appended. + * ID: "append-dataset" + * Name: "Append experimental dataset" + */ APPEND_OVERWRITE("append-dataset", "Append experimental dataset"); private String id; private String name; + /** + * Constructor for the Workflow enum to initialize ID and Name. + * @param id The ID of the workflow. + * @param name The name of the workflow. + */ Workflow(String id, String name) { - this.id = id; this.name = name; } + /** + * Get the ID of the workflow. + * @return The ID of the workflow. + */ public String getId() { return id; } - public String getName() { return name; } + /** + * Get the name of the workflow. + * @return The name of the workflow. + */ + public String getName() { + return name; + } + + /** + * Check if the given value is equal to the ID of the workflow. + * @param value The value to compare with the workflow ID. + * @return true if the value is equal to the ID, false otherwise. + */ public boolean isEqual(String value) { return Optional.ofNullable(id.equals(value)).orElse(false); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 759d91328..df529af99 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -65,8 +65,19 @@ public AppendOverwritePhenotypesWorkflow(AppendOverwriteIDValidation expUnitIDVa this.brapiCommitMiddleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link(brAPICommit); } + /** + * Processes the import workflow based on the provided import service context. + * If the provided context is not valid or if the workflow is not equal to the context workflow, returns an empty Optional. + * If the context is null, returns a no-preview result with metadata for this workflow. + * Processes the import preview using middleware, catches and handles any processing errors, and builds the import preview response. + * Updates status based on the processing and commit actions if applicable. + * + * @param context The import service context containing upload, data, program, user, commit flag, and workflow information. + * @return Optional containing ImportWorkflowResult with workflow metadata and import preview response if successful, else empty Optional. + */ @Override public Optional process(ImportServiceContext context) { + // Metadata about this workflow processing the context ImportWorkflow workflow = ImportWorkflow.builder() .id(getWorkflow().getId()) @@ -109,7 +120,7 @@ public Optional process(ImportServiceContext context) { // Stop and return any errors that occurred while processing Optional previewException = Optional.ofNullable(processedPreviewContext.getAppendOverwriteWorkflowContext().getProcessError()); - if (previewException.isPresent() ) { + if (previewException.isPresent()) { log.debug(String.format("%s in %s", previewException.get().getException().getClass().getName(), previewException.get().getLocalTransactionName())); result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.ofNullable(previewException.get().getException()))); return result; @@ -144,7 +155,7 @@ public Optional process(ImportServiceContext context) { AppendOverwriteMiddlewareContext brapiCommittedContext = this.brapiCommitMiddleware.process(processedPreviewContext); Optional brapiCommitException = Optional.ofNullable(brapiCommittedContext.getAppendOverwriteWorkflowContext().getProcessError()); - if (brapiCommitException.isPresent() ) { + if (brapiCommitException.isPresent()) { log.debug(String.format("%s in %s", brapiCommitException.get().getException().getClass()), brapiCommitException.get().getLocalTransactionName()); result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.ofNullable(brapiCommitException.get().getException()))); return result; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java index 2024ec07d..642e46a97 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java @@ -40,69 +40,157 @@ public BrAPICreationFactory(PendingEntityFactory pendingEntityFactory) { this.pendingEntityFactory = pendingEntityFactory; } + /** + * Creates a workflow for creating a BrAPI trial. + * + * @param context The AppendOverwriteMiddlewareContext containing the necessary information for the creation of the trial. + * @param pendingEntityFactory The factory responsible for creating pending entities. + * @return A WorkflowCreation instance for creating a BrAPITrial. + */ public static WorkflowCreation trialWorkflowCreation(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowCreation(pendingEntityFactory.pendingTrialBean(context)); } + /** + * Creates a workflow for creating a BrAPI dataset. + * + * @param context The AppendOverwriteMiddlewareContext containing the necessary information for the creation of the dataset. + * @param pendingEntityFactory The factory responsible for creating pending entities. + * @return A WorkflowCreation instance for creating a BrAPIListDetails. + */ public static WorkflowCreation datasetWorkflowCreation(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowCreation(pendingEntityFactory.pendingDatasetBean(context)); } + /** + * Creates a workflow for creating a BrAPI study. + * + * @param context The AppendOverwriteMiddlewareContext containing the necessary information for the creation of the study. + * @param pendingEntityFactory The factory responsible for creating pending entities. + * @return A WorkflowCreation instance for creating a BrAPIStudy. + */ public static WorkflowCreation studyWorkflowCreation(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowCreation(pendingEntityFactory.pendingStudyBean(context)); } + /** + * Creates a workflow for creating a BrAPI observation. + * + * @param context The AppendOverwriteMiddlewareContext containing the necessary information for the creation of the observation. + * @param pendingEntityFactory The factory responsible for creating pending entities. + * @return A WorkflowCreation instance for creating a BrAPIObservation. + */ public static WorkflowCreation observationWorkflowCreation(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowCreation(pendingEntityFactory.pendingObservationBean(context)); } + /** + * Creates a workflow for creating a BrAPI observation unit. + * + * @param context The AppendOverwriteMiddlewareContext containing the necessary information for the creation of the observation unit. + * @param pendingEntityFactory The factory responsible for creating pending entities. + * @return A WorkflowCreation instance for creating a BrAPIObservationUnit. + */ public static WorkflowCreation observationUnitWorkflowCreation(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowCreation(pendingEntityFactory.pendingObservationUnitBean(context)); } + /** + * Creates a workflow for creating a Program Location. + * + * @param context The AppendOverwriteMiddlewareContext containing the necessary information for the creation of the location. + * @param pendingEntityFactory The factory responsible for creating pending entities. + * @return A WorkflowCreation instance for creating a ProgramLocation. + */ public static WorkflowCreation locationWorkflowCreation(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { return new WorkflowCreation(pendingEntityFactory.pendingLocationBean(context)); } + /** + * This method is a Spring bean that creates a prototype instance of a WorkflowCreation object for BrAPITrial entities. + * The WorkflowCreation object is responsible for creating a workflow for a BrAPITrial entity. + * + * @param context The middleware context containing information required for creating the workflow. + * @return A WorkflowCreation object specialized for BrAPITrial entities. + */ @Bean @Prototype public WorkflowCreation trialWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { return trialWorkflowCreation(context, pendingEntityFactory); } + /** + * This method creates a new instance of WorkflowCreation for handling dataset-related tasks within the BrAPIListDetails context. + * The WorkflowCreation instance will be configured as a Prototype bean, meaning it will return a new instance each time it is requested. + * The method takes an AppendOverwriteMiddlewareContext as input, which provides the necessary context for the workflow creation. + * The method returns a WorkflowCreation instance for handling dataset creation operations within the BrAPIListDetails context. + * @param context The AppendOverwriteMiddlewareContext providing the context for the workflow creation process. + * @return A WorkflowCreation instance configured to handle dataset creation operations within the BrAPIListDetails context. + */ @Bean @Prototype public WorkflowCreation datasetWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { return datasetWorkflowCreation(context, pendingEntityFactory); } + /** + * This method creates a bean for creating a workflow for a BrAPI study. + * The bean is of prototype scope, meaning that a new instance of the bean will be created every time it is requested. + * The workflow creation is specific to BrAPIStudy type. + * + * @param context The AppendOverwriteMiddlewareContext object containing the context information for workflow creation. + * @return a WorkflowCreation object specialized for BrAPIStudy, configured using the provided context and pendingEntityFactory. + */ @Bean @Prototype public WorkflowCreation studyWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { return studyWorkflowCreation(context, pendingEntityFactory); } + /** + * This method creates a new instance of WorkflowCreation for BrAPIObservation objects based on the provided AppendOverwriteMiddlewareContext. + * It is marked as a prototype bean, meaning a new instance will be created each time it is injected. + * + * @param context The AppendOverwriteMiddlewareContext containing the necessary information for the workflow creation. + * @return A WorkflowCreation instance for BrAPIObservation objects. + */ @Bean @Prototype public WorkflowCreation observationWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { return observationWorkflowCreation(context, pendingEntityFactory); } + /** + * This method is responsible for creating a new instance of WorkflowCreation for BrAPIObservationUnit. + * It leverages the observationUnitWorkflowCreation method to initialize the WorkflowCreation object. + * + * @param context the AppendOverwriteMiddlewareContext object containing relevant context information + * @return a new instance of WorkflowCreation object + * + */ @Bean @Prototype public WorkflowCreation observationUnitWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { return observationUnitWorkflowCreation(context, pendingEntityFactory); } + /** + * This method creates a new WorkflowCreation instance for handling ProgramLocation objects by appending or overwriting the entities. + * The WorkflowCreation instance is specific to each invocation. + * + * @param context the AppendOverwriteMiddlewareContext object containing the context for workflow creation + * @return a WorkflowCreation instance configured for handling ProgramLocation objects + */ @Bean @Prototype public WorkflowCreation locationWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { return locationWorkflowCreation(context, pendingEntityFactory); } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java index 0af59ba54..b69a4ddfe 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java @@ -30,37 +30,57 @@ import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; import static org.breedinginsight.dao.db.enums.DataType.DATE; +/** + * This class represents a DateValidator that implements the ObservationValidator interface. + * It is responsible for validating date fields in observations based on specific criteria. + */ @Slf4j @Singleton public class DateValidator implements ObservationValidator { + @Inject ObservationService observationService; + private final String dateMessage = "Incorrect date format detected. Expected YYYY-MM-DD"; private final String dateTimeMessage = "Incorrect datetime format detected. Expected YYYY-MM-DD or YYYY-MM-DDThh:mm:ss+hh:mm"; + /** + * Constructor for the DateValidator class that initializes the ObservationService instance. + * @param observationService An instance of ObservationService used for validation. + */ public DateValidator(ObservationService observationService) { this.observationService = observationService; } + + /** + * Validates a specific field in an observation based on the value and traits provided. + * + * @param fieldName The name of the field to validate. + * @param value The value of the field to be validated. + * @param variable The trait variable associated with the field. + * @return An Optional containing a ValidationError if validation fails, or Optional.empty() if validation passes. + */ @Override public Optional validateField(String fieldName, String value, Trait variable) { + // Check if the observation is blank if (observationService.isBlankObservation(value)) { - log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); + log.debug(String.format("Skipping validation of observation because there is no value.\n\tVariable: %s", fieldName)); return Optional.empty(); } + // Check if the observation is NA if (observationService.isNAObservation(value)) { - log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); + log.debug(String.format("Skipping validation of observation because it is NA.\n\tVariable: %s", fieldName)); return Optional.empty(); } - // Is this a timestamp field? + // Check if the field is a timestamp if (fieldName.startsWith(TIMESTAMP_PREFIX)) { if (!observationService.validDateValue(value) && !observationService.validDateTimeValue(value)) { return Optional.of(new ValidationError(fieldName, dateTimeMessage, HttpStatus.UNPROCESSABLE_ENTITY)); } } else { - // Skip if there is no trait data if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { return Optional.empty(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/FieldValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/FieldValidator.java index 2747d9749..d4ec36e01 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/FieldValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/FieldValidator.java @@ -25,15 +25,35 @@ import java.util.List; import java.util.Optional; +/** + * This class represents a FieldValidator that implements ObservationValidator interface to validate fields. + * FieldValidator is a Primary and Singleton bean in the application. + */ @Primary @Singleton public class FieldValidator implements ObservationValidator { + + /** + * List of ObservationValidator instances to perform validation on fields. + */ private final List validators; + /** + * Constructor for FieldValidator which accepts a list of ObservationValidator instances. + * @param validators List of ObservationValidator instances + */ public FieldValidator(List validators) { this.validators = validators; } + /** + * Validates a field by applying validation from multiple validators in the list. + * Returns the first validation error encountered, if any. + * @param fieldName The name of the field being validated + * @param value The value of the field being validated + * @param variable The trait variable associated with the field + * @return Optional<ValidationError> Optional containing the first validation error encountered, if any + */ @Override public Optional validateField(String fieldName, String value, Trait variable) { return validators.stream() diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NominalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NominalValidator.java index ab390d2bf..1bc0fa3e5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NominalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NominalValidator.java @@ -30,24 +30,44 @@ import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; import static org.breedinginsight.dao.db.enums.DataType.NOMINAL; +/** + * This class represents a NominalValidator which implements the ObservationValidator interface. + * It is responsible for validating nominal fields within observations. + */ @Slf4j @Singleton public class NominalValidator implements ObservationValidator { + @Inject ObservationService observationService; + /** + * Constructor for NominalValidator class that takes an ObservationService as a parameter. + * @param observationService the ObservationService used for validation + */ public NominalValidator(ObservationService observationService) { this.observationService = observationService; } + + /** + * Validates a field within an observation for nominal data. + * + * @param fieldName the name of the field being validated + * @param value the value of the field being validated + * @param variable the Trait variable associated with the field + * @return an Optional containing a ValidationError if validation fails, otherwise an empty Optional + */ @Override public Optional validateField(String fieldName, String value, Trait variable) { + // Skip validation if observation is blank if (observationService.isBlankObservation(value)) { - log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); + log.debug(String.format("Skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); return Optional.empty(); } + // Skip validation if observation is NA if (observationService.isNAObservation(value)) { - log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); + log.debug(String.format("Skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); return Optional.empty(); } @@ -61,7 +81,7 @@ public Optional validateField(String fieldName, String value, T return Optional.empty(); } - // Skip if this is not an ordinal trait + // Skip if this is not a nominal trait if (!NOMINAL.equals(variable.getScale().getDataType())) { return Optional.empty(); } @@ -73,4 +93,4 @@ public Optional validateField(String fieldName, String value, T return Optional.empty(); } -} +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java index d3d2b74ce..5fb3b9480 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java @@ -31,58 +31,76 @@ import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; import static org.breedinginsight.dao.db.enums.DataType.NUMERICAL; +/** + * NumericalValidator class is responsible for validating numerical observations against specified constraints. + * This class implements the ObservationValidator interface. + * + * The class provides a validateField method which takes the field name, value, and trait as inputs and + * returns an Optional containing a ValidationError if any validation error is encountered, or Optional.empty() if + * the validation passes successfully. + */ @Slf4j @Singleton public class NumericalValidator implements ObservationValidator { + @Inject ObservationService observationService; + /** + * Constructor for NumericalValidator class. + * @param observationService An instance of ObservationService to perform observation-related operations. + */ public NumericalValidator(ObservationService observationService) { this.observationService = observationService; } + + /** + * Validates a numerical observation against specified constraints. + * Returns an Optional containing a ValidationError if validation fails, or Optional.empty() if successful. + * @param fieldName The name of the observation field to be validated. + * @param value The value of the observation to be validated. + * @param variable The trait associated with the observation. + * @return Optional containing a ValidationError if validation fails, or Optional.empty() if successful. + */ @Override public Optional validateField(String fieldName, String value, Trait variable) { if (observationService.isBlankObservation(value)) { - log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); + log.debug(String.format("Skipping validation of observation because there is no value.\n\tVariable: %s", fieldName)); return Optional.empty(); } if (observationService.isNAObservation(value)) { - log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); + log.debug(String.format("Skipping validation of observation because it is NA.\n\tVariable: %s", fieldName)); return Optional.empty(); } - // Skip if field is a timestamp + // Skip validation if field is a timestamp if (fieldName.startsWith(TIMESTAMP_PREFIX)) { return Optional.empty(); } - // Skip if there is no trait data - if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { + // Skip validation if there is no trait data or if the trait is not numerical + if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null || + !NUMERICAL.equals(variable.getScale().getDataType())) { return Optional.empty(); } - // Skip if this is not a numerical trait - if (!NUMERICAL.equals(variable.getScale().getDataType())) { - return Optional.empty(); - } - - // Make new validation error if the value is non-numeric + // Check if the value is a valid numeric value Optional number = observationService.validNumericValue(value); if (number.isEmpty()) { return Optional.of(new ValidationError(fieldName, "Non-numeric text in a numerical field", HttpStatus.UNPROCESSABLE_ENTITY)); } + // Perform range validation for numeric value Optional validationError = number .flatMap(num -> { if (observationService.validNumericRange(num, variable.getScale())) { - return Optional.empty(); // Return empty Optional if value is in numeric range + return Optional.empty(); // Return empty Optional if value is within numeric range } else { return Optional.of(new ValidationError(fieldName, "Value outside of min/max range detected", HttpStatus.UNPROCESSABLE_ENTITY)); } }); return validationError; - } -} +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/OrdinalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/OrdinalValidator.java index 4ea8bdead..e961f6871 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/OrdinalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/OrdinalValidator.java @@ -30,38 +30,57 @@ import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; import static org.breedinginsight.dao.db.enums.DataType.ORDINAL; +/** + * This class represents a validator specifically designed for ordinal traits. + * It implements the ObservationValidator interface. + */ @Slf4j @Singleton public class OrdinalValidator implements ObservationValidator { + @Inject ObservationService observationService; + /** + * Constructs an instance of OrdinalValidator with the specified ObservationService. + * + * @param observationService the ObservationService used for validation + */ public OrdinalValidator(ObservationService observationService) { this.observationService = observationService; } + + /** + * Validates a field for ordinal traits. + * + * @param fieldName the name of the field being validated + * @param value the value of the field + * @param variable the trait related to the field + * @return an Optional containing a ValidationError if validation fails, empty otherwise + */ @Override public Optional validateField(String fieldName, String value, Trait variable) { if (observationService.isBlankObservation(value)) { - log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); + log.debug(String.format("Skipping validation of observation because there is no value.\n\tVariable: %s", fieldName)); return Optional.empty(); } if (observationService.isNAObservation(value)) { - log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); + log.debug(String.format("Skipping validation of observation because it is NA.\n\tVariable: %s", fieldName)); return Optional.empty(); } - // Skip if field is a timestamp + // Skip validation if field is a timestamp if (fieldName.startsWith(TIMESTAMP_PREFIX)) { return Optional.empty(); } - // Skip if there is no trait data + // Skip validation if there is no trait data if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { return Optional.empty(); } - // Skip if this is not an ordinal trait + // Skip validation if this is not an ordinal trait if (!ORDINAL.equals(variable.getScale().getDataType())) { return Optional.empty(); } From 68afa68a82fc34e7c4df5819132ac6ea2e45b429 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:02:36 -0400 Subject: [PATCH 177/203] configure maven build github action to be manually triggerable --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 59a43f17e..1ddbede0a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,6 +18,7 @@ name: maven build on: + workflow_dispatch: pull_request: type: [opened, edited] From 9fa39f6f9a23713c3d93d739e5d6325c08477b24 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 2 Jul 2024 18:08:03 -0400 Subject: [PATCH 178/203] add debug statements to experiment file import tests --- .../importer/ExperimentFileImportTest.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 86d7a27e9..dfd4a2020 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -28,6 +28,7 @@ import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.typeAdapters.PaginationTypeAdapter; import org.brapi.v2.model.BrAPIExternalReference; @@ -83,6 +84,7 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.*; +@Slf4j @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @@ -203,6 +205,7 @@ public String getExperimentWorkflowId(int workflowIndex) { @Test @SneakyThrows public void importNewExpNewLocNoObsSuccess() { + log.debug("importNewExpNewLocNoObsSuccess"); Program program = createProgram("New Exp and Loc", "NEXPL", "NEXPL", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); Map validRow = new HashMap<>(); validRow.put(Columns.GERMPLASM_GID, "1"); @@ -248,6 +251,7 @@ public void importNewExpNewLocNoObsSuccess() { @Test @SneakyThrows public void importNewExpMultiNewEnvSuccess() { + log.debug("importNewExpMultiNewEnvSucces"); Program program = createProgram("New Exp and Multi New Env", "MULENV", "MULENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); Map firstEnv = new HashMap<>(); firstEnv.put(Columns.GERMPLASM_GID, "1"); @@ -316,6 +320,7 @@ public void importNewExpMultiNewEnvSuccess() { @Test @SneakyThrows public void importExistingExpAndEnvErrorMessage() { + log.debug("importExistingExpAndEnvErrorMessage"); Program program = createProgram("New Env Existing Exp", "DUPENV", "DUPENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); @@ -366,6 +371,7 @@ public void importExistingExpAndEnvErrorMessage() { @Test @SneakyThrows public void importNewEnvNoObsSuccess() { + log.debug("importNewEnvNoObsSuccess"); Program program = createProgram("New Env", "NEWENV", "NEWENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); Map newEnv = new HashMap<>(); @@ -401,6 +407,7 @@ public void importNewEnvNoObsSuccess() { @ValueSource(booleans = {true, false}) @SneakyThrows public void verifyMissingDataThrowsError(boolean commit) { + log.debug("verifyMissingDataThrowsError"); Program program = createProgram("Missing Req Cols "+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); Map base = new HashMap<>(); @@ -470,6 +477,7 @@ public void verifyMissingDataThrowsError(boolean commit) { @Test @SneakyThrows public void importNewExpWithObsVar() { + log.debug("importNewExpWithObsVar"); List traits = importTestUtils.createTraits(1); Program program = createProgram("New Exp with Observations Vars", "EXPVRR", "EXPVRR", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -508,6 +516,7 @@ public void importNewExpWithObsVar() { @ValueSource(booleans = {true, false}) @SneakyThrows public void verifyDiffYearSameEnvThrowsError(boolean commit) { + log.debug("verifyDiffYEarSameEnvThrowsError"); Program program = createProgram("Diff Years "+(commit ? "C" : "P"), "YEARS"+(commit ? "C" : "P"), "YEARS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(2), null); List> rows = new ArrayList<>(); @@ -548,6 +557,7 @@ public void verifyDiffYearSameEnvThrowsError(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void verifyDiffLocSameEnvThrowsError(boolean commit) { + log.debug("verifyDiffLocSameEnvThrowsError"); Program program = createProgram("Diff Locations "+(commit ? "C" : "P"), "LOCS"+(commit ? "C" : "P"), "LOCS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(2), null); List> rows = new ArrayList<>(); @@ -587,6 +597,7 @@ public void verifyDiffLocSameEnvThrowsError(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void importNewExpWithObs(boolean commit) { + log.debug("importNewExpWithObs"); List traits = importTestUtils.createTraits(1); Program program = createProgram("New Exp with Observations "+(commit ? "C" : "P"), "NEXOB"+(commit ? "C" : "P"), "NEXOB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -628,6 +639,7 @@ public void importNewExpWithObs(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { + log.debug("verifyFailureImportNewExpWithInvalidObs"); List traits = importTestUtils.createTraits(1); Program program = createProgram("Invalid Observations "+(commit ? "C" : "P"), "INVOB"+(commit ? "C" : "P"), "INVOB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -655,6 +667,7 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void verifyFailureNewOuExistingEnv(boolean commit) { + log.debug("verifyFailureNewOuExistingEnv"); Program program = createProgram("New OU Existing Env "+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); @@ -697,6 +710,7 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { @Test @SneakyThrows public void importNewObsVarExistingOu() { + log.debug("importNewObsVarExistingOu"); List traits = importTestUtils.createTraits(2); Program program = createProgram("New ObsVar Existing OU", "OUVAR", "OUVAR", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -761,6 +775,7 @@ public void importNewObsVarExistingOu() { @Test @SneakyThrows public void importNewObsVarByObsUnitId() { + log.debug("importNewObsVarByObsUnitId"); List traits = importTestUtils.createTraits(2); Program program = createProgram("New ObsVar Referring to OU by ID", "OUVAR", "VAROU", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -813,6 +828,7 @@ public void importNewObsVarByObsUnitId() { @ValueSource(booleans = {true, false}) @SneakyThrows public void importNewObservationDataByObsUnitId(boolean commit) { + log.debug("importNewObservationDataByObsUnitId"); List traits = importTestUtils.createTraits(1); Program program = createProgram("New Observation Referring to OU by ID"+(commit ? "C" : "P"), "OUDAT"+(commit ? "C" : "P"), "DATOU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -883,6 +899,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void importNewObsExistingOu(boolean commit) { + log.debug("importNewObsExistingOu"); List traits = importTestUtils.createTraits(1); Program program = createProgram("New Obs Existing OU "+(commit ? "C" : "P"), "OUOBS"+(commit ? "C" : "P"), "OUOBS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -949,6 +966,7 @@ public void importNewObsExistingOu(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { + log.debug("verifyFailureImportNewObsExistingOuWithExistingObs"); List traits = importTestUtils.createTraits(1); Program program = createProgram("New Obs Existing Obs "+(commit ? "C" : "P"), "FEXOB"+(commit ? "C" : "P"), "FEXOB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -1008,6 +1026,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { @Test @SneakyThrows public void importSecondExpAfterFirstExpWithObs() { + log.debug("importSecondExpAfterFirstExpWithObs"); List traits = importTestUtils.createTraits(1); Program program = createProgram("New Exp After First", "NEAF", "NEAF", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExpA = new HashMap<>(); @@ -1079,6 +1098,7 @@ public void importSecondExpAfterFirstExpWithObs() { @ValueSource(booleans = {true, false}) @SneakyThrows public void importNewObsAfterFirstExpWithObs(boolean commit) { + log.debug("importNewObsAfterFirstExpWithObs"); List traits = importTestUtils.createTraits(2); Program program = createProgram("Exp with additional Uploads "+(commit ? "C" : "P"), "EXAU"+(commit ? "C" : "P"), "EXAU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -1153,6 +1173,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { + log.debug("importNewObsAfterFirstExpWithObs_blank"); List traits = importTestUtils.createTraits(2); Program program = createProgram("Exp with additional Uploads (blank) "+(commit ? "C" : "P"), "EXAUB"+(commit ? "C" : "P"), "EXAUB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); From cad35dc5caa67a8ff1cae80ff48e51ebbfc09b24 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 2 Jul 2024 18:32:24 -0400 Subject: [PATCH 179/203] add debug statements to ProgramCache#populate --- src/main/java/org/breedinginsight/daos/cache/ProgramCache.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java b/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java index d231116ae..5de4e7344 100644 --- a/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java +++ b/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java @@ -122,7 +122,9 @@ public void populate(@NotNull UUID key) { } finally { log.debug("Releasing semaphore: " + cacheKey); connection.getAtomicLong(cacheKey+":refreshing").set(0); + log.debug("Fetched connection: " + cacheKey); semaphore.release(); + log.debug("connection released"); } }); } From 4051b44bb97fbe6cb56a8eb42da7c47ebfb164f5 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 2 Jul 2024 19:09:29 -0400 Subject: [PATCH 180/203] add debug statements to sample submission file import test --- .../brapps/importer/SampleSubmissionFileImportTest.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java index 8c5b308c3..31eafc173 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java @@ -28,6 +28,7 @@ import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.typeAdapters.PaginationTypeAdapter; import org.brapi.v2.model.BrAPIExternalReference; @@ -81,6 +82,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +@Slf4j @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @@ -169,6 +171,7 @@ public void setup() { @Test @SneakyThrows public void importGIDSuccess() { + log.debug("importGIDSuccess"); Program program = createProgram("Import GID Success", "GIDS", "GIDS", BRAPI_REFERENCE_SOURCE, createGermplasm(96), null); List> validFile = new ArrayList<>(); @@ -215,6 +218,7 @@ public void importGIDSuccess() { @Test @SneakyThrows public void importObsUnitIdSuccess() { + log.debug("importObsUnitIdSuccess"); Program program = createProgram("Import ObsUnitID success", "OBSID", "OBSID", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); var experimentId = createExperiment(program); @@ -265,6 +269,7 @@ public void importObsUnitIdSuccess() { @ValueSource(booleans = {true, false}) @SneakyThrows public void importMissingGIDAndObsUnitIdFailure(boolean commit) { + log.debug("importMissingGIDAndObsUnitIdFailure"); Program program = createProgram("Missing GID/ObsUnit ID " + (commit ? "C" : "P"), "MGIOB"+ (commit ? "C" : "P"), "MGIOB"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, null, null); List> validFile = new ArrayList<>(); @@ -288,6 +293,7 @@ public void importMissingGIDAndObsUnitIdFailure(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void verifyMissingDataThrowsError(boolean commit) { + log.debug("verifyMissingDataThrowsError"); Program program = createProgram("Missing Req Cols "+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(96), null); Map base = new HashMap<>(); base.put(Columns.PLATE_ID, "valid_1"); @@ -312,6 +318,7 @@ public void verifyMissingDataThrowsError(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void importInvalidGIDFailure(boolean commit) { + log.debug("importInvalidGIDFailure"); Program program = createProgram("Invalid GID " + (commit ? "C" : "P"), "INGID"+ (commit ? "C" : "P"), "INGID"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, null, null); List> validFile = new ArrayList<>(); @@ -335,6 +342,7 @@ public void importInvalidGIDFailure(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void importInvalidObsUnitIdFailure(boolean commit) { + log.debug("importInvalidObsUnitIdFailure"); Program program = createProgram("Invalid ObsUnit ID " + (commit ? "C" : "P"), "INOBS"+ (commit ? "C" : "P"), "INOBS"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, null, null); List> validFile = new ArrayList<>(); @@ -358,6 +366,7 @@ public void importInvalidObsUnitIdFailure(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void importConflictingWellsFailure(boolean commit) { + log.debug("importConflictingWellsFailure"); Program program = createProgram("Conflicting Wells " + (commit ? "C" : "P"), "WELL"+ (commit ? "C" : "P"), "WELL"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(2), null); List> validFile = new ArrayList<>(); From 16871fc4f150fdbbb3bd08cb41170eaaae4fd8d6 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 3 Jul 2024 12:55:33 -0400 Subject: [PATCH 181/203] [BI-2134] - added debug logging --- .../org/breedinginsight/brapps/importer/ImportTestUtils.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index 84da9b536..f465e6d2d 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -28,6 +28,7 @@ import io.micronaut.http.client.multipart.MultipartBody; import io.micronaut.http.netty.cookies.NettyCookie; import io.reactivex.Flowable; +import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.model.v1.request.ProgramRequest; import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.api.v1.controller.TestTokenValidator; @@ -57,6 +58,7 @@ * * To use, instantiate a new instance of this class, then use it like a regular static utility class */ +@Slf4j public class ImportTestUtils { Pattern UUID_REGEX = Pattern.compile("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"); @@ -139,6 +141,7 @@ public HttpResponse getUploadedFile(String importId, RxHttpClient client if (response.getStatus().equals(HttpStatus.ACCEPTED)) { Thread.sleep(1000); + log.debug("202 Accepted response. Sleeping for 1000ms."); return getUploadedFile(importId, client, program, mappingId); } else { return response; From b4aedc3378ca76e282e0d38dec6509e6ac2537a7 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 3 Jul 2024 15:44:26 -0400 Subject: [PATCH 182/203] add create-new-experiment workflow id to SampleSubmissionFileImportTest#createExperiment --- .../model/v1/response/ValidationErrors.java | 6 +++-- .../importer/ExperimentFileImportTest.java | 24 ++--------------- .../importer/GermplasmFileImportTest.java | 1 + .../brapps/importer/ImportTestUtils.java | 27 +++++++++++++++++++ .../SampleSubmissionFileImportTest.java | 9 ++++--- 5 files changed, 40 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java b/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java index 923529505..7fdd58347 100644 --- a/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java +++ b/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java @@ -17,8 +17,9 @@ package org.breedinginsight.api.model.v1.response; -import io.micronaut.http.HttpStatus; -import lombok.*; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; import lombok.experimental.Accessors; import java.util.ArrayList; @@ -47,6 +48,7 @@ public void addError(Integer rowNumber, ValidationError validationError){ newRow.addError(validationError); rowErrors.add(newRow); } + public void merge(ValidationErrors validationErrors){ for (RowValidationErrors rowValidationErrors: validationErrors.getRowErrors()){ for (ValidationError validationError: rowValidationErrors.getErrors()) { diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index dfd4a2020..7171155d0 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -162,30 +162,10 @@ public void setup() { /** * Implicit test that the workflow ids are assigned in the following order */ - newExperimentWorkflowId = getExperimentWorkflowId(0); - appendOverwriteWorkflowId = getExperimentWorkflowId(1); - + newExperimentWorkflowId = importTestUtils.getExperimentWorkflowId(client, 0); + appendOverwriteWorkflowId = importTestUtils.getExperimentWorkflowId(client, 1); } - /** - * TODO: assumes new workflow is first in list, doesn't look at position property, would be more robust to - * look at that instead of assuming order - * @return - */ - public String getExperimentWorkflowId(int workflowIndex) { - - // GET /import/mappings{?importName} - Flowable> call = client.exchange( - GET("/import/mappings/"+mappingId+"/workflows").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - HttpResponse response = call.blockingFirst(); - JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); - - return JsonParser.parseString(response.body()).getAsJsonObject() - .getAsJsonObject("result") - .getAsJsonArray("data") - .get(workflowIndex).getAsJsonObject().get("id").getAsString(); - } /* Tests - new experiment diff --git a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java index 35fe42946..30bb1830d 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java @@ -81,6 +81,7 @@ public class GermplasmFileImportTest extends BrAPITest { (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .create(); + @BeforeAll public void setup() { importTestUtils = new ImportTestUtils(); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index f465e6d2d..1c7b4acbb 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -190,7 +190,34 @@ public Map setup(RxHttpClient client, Gson gson, DSLContext dsl, "securityFp", securityFp); } + /** + * TODO: assumes new workflow is first in list, doesn't look at position property, would be more robust to + * look at that instead of assuming order + * @return + */ + public String getExperimentWorkflowId(RxHttpClient client, int workflowIndex) { + // Get the mapping id for experiment imports + Flowable> mappingCall = client.exchange( + GET("/import/mappings?importName=ExperimentsTemplateMap").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class); + HttpResponse mappingResponse = mappingCall.blockingFirst(); + String mappingId = JsonParser.parseString(mappingResponse.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + // Get the workflow id for the workflow at position workflowIndex in the collection of all available experiment workflows + // GET /import/mappings{?importName} + Flowable> call = client.exchange( + GET("/import/mappings/"+mappingId+"/workflows").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + return JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(workflowIndex).getAsJsonObject().get("id").getAsString(); + } public JsonObject uploadAndFetch(File file, Map userData, Boolean commit, RxHttpClient client, Program program, String mappingId) throws InterruptedException { Flowable> call = uploadDataFile(file, userData, commit, client, program, mappingId); diff --git a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java index 31eafc173..71bf6197c 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java @@ -148,6 +148,8 @@ public class SampleSubmissionFileImportTest extends BrAPITest { .registerTypeAdapter(BrAPIPagination.class, new PaginationTypeAdapter()) .create(); + private String newExperimentWorkflowId; + @BeforeAll public void setup() { importTestUtils = new ImportTestUtils(); @@ -155,7 +157,7 @@ public void setup() { mappingId = (String) setupObjects.get("mappingId"); testUser = (BiUserEntity) setupObjects.get("testUser"); securityFp = (FannyPack) setupObjects.get("securityFp"); - + newExperimentWorkflowId = importTestUtils.getExperimentWorkflowId(client, 0); } /* @@ -565,13 +567,14 @@ private String createExperiment(Program program) throws IOException, Interrupted .getAsJsonArray("data") .get(0).getAsJsonObject().get("id").getAsString(); - JsonObject importResult = importTestUtils.uploadAndFetch( + JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null), null, true, client, program, - expMappingId); + expMappingId, + newExperimentWorkflowId); return importResult .get("preview").getAsJsonObject() .get("rows").getAsJsonArray() From 2d3eaa5fefaafb0e3064b6b5b664619a0412f6e9 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 4 Jul 2024 18:00:59 -0400 Subject: [PATCH 183/203] remove manual trigger from maven build github action --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1ddbede0a..59a43f17e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,6 @@ name: maven build on: - workflow_dispatch: pull_request: type: [opened, edited] From 50a53987abf0456a0fa5111220d76e0307bf7048 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 4 Jul 2024 18:16:54 -0400 Subject: [PATCH 184/203] add documentation string --- .../service/ObservationVariableService.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java index 53bb9b745..ce43db2b4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java @@ -75,14 +75,28 @@ public List fetchTraitsByName(Set varNames, Program program) thro return traits; } + /** + * Validates that each timestamp column corresponds to a phenotype column. + * + * This method takes a Set of observationVariableNames and a List of timestamp columns, and checks + * if each timestamp column corresponds to a phenotype column by comparing it with the observationVariableNames. + * + * @param observationVariableNames A Set of observation variable names representing the phenotype columns. + * @param timestampCols A List of timestamp columns to be validated. + * + * @return An Optional containing a list of ValidationErrors if there are mismatches, or an empty + * Optional if all timestamp columns have corresponding phenotype columns. + */ public Optional> validateMatchedTimestamps(Set observationVariableNames, List> timestampCols) { Optional> ve = Optional.empty(); - // Check that each ts column corresponds to a phenotype column + + // Check that each timestamp column corresponds to a phenotype column List valErrs = timestampCols.stream() .filter(col -> !(observationVariableNames.contains(col.name().replaceFirst(ExpImportProcessConstants.TIMESTAMP_REGEX, StringUtils.EMPTY)))) .map(col -> new ValidationError(col.name().toString(), String.format("Timestamp column %s lacks corresponding phenotype column", col.name().toString()), HttpStatus.UNPROCESSABLE_ENTITY)) .collect(Collectors.toList()); + if (valErrs.size() > 0) { ve = Optional.of(valErrs); } From 5846513587d0d96447e5634cff4905f30624502e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 4 Jul 2024 18:27:16 -0400 Subject: [PATCH 185/203] changes type key to supported types key in github action --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 59a43f17e..da9e70b3f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ name: maven build on: pull_request: - type: [opened, edited] + types: [opened, edited] jobs: build: From 09640e86f9a04c6e1b33e3b2d1cd3137ce4c0ab7 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 4 Jul 2024 18:30:51 -0400 Subject: [PATCH 186/203] add documentation string --- .../service/ObservationService.java | 52 +++++++++++++------ 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index 628eca472..a6ecac2c0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -121,20 +121,39 @@ public OffsetDateTime parseDateTime(String dateString) { } } + /** + * Constructs a new BrAPI observation based on the provided parameters. + * + * @param commit boolean value indicating whether the operation should be committed + * @param germplasmName the name of the germplasm associated with the observation + * @param variableName the name of the observation variable + * @param study the BrAPI study object to associate with the observation + * @param seasonDbId the unique identifier of the season for the observation + * @param obsUnit the BrAPI observation unit object + * @param value the value of the observation + * @param trialId the identifier of the trial associated with the observation + * @param studyId the identifier of the study associated with the observation + * @param obsUnitId the identifier of the observation unit associated with the observation + * @param observationId the identifier of the observation + * @param referenceSource the source of the reference + * @param user the User object representing the creator of the observation + * @param program the Program object associated with the observation + * @return a newly constructed BrAPIObservation object + */ public BrAPIObservation constructNewBrAPIObservation(boolean commit, - String germplasmName, - String variableName, - BrAPIStudy study, - String seasonDbId, - BrAPIObservationUnit obsUnit, - String value, - UUID trialId, - UUID studyId, - UUID obsUnitId, - UUID observationId, - String referenceSource, - User user, - Program program) { + String germplasmName, + String variableName, + BrAPIStudy study, + String seasonDbId, + BrAPIObservationUnit obsUnit, + String value, + UUID trialId, + UUID studyId, + UUID obsUnitId, + UUID observationId, + String referenceSource, + User user, + Program program) { BrAPIObservation observation = new BrAPIObservation(); observation.setGermplasmName(germplasmName); @@ -145,20 +164,21 @@ public BrAPIObservation constructNewBrAPIObservation(boolean commit, observation.setObservationUnitName(obsUnit.getObservationUnitName()); observation.setValue(value); - // The BrApi server needs this. Breedbase does not. + // The BrApi server needs this. Breedbase does not. BrAPISeason season = new BrAPISeason(); season.setSeasonDbId(seasonDbId); observation.setSeason(season); - if(commit) { + if (commit) { Map createdBy = new HashMap<>(); createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_ID, user.getId()); createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_NAME, user.getName()); observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_BY, createdBy); observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_DATE, DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(OffsetDateTime.now())); - observation.setExternalReferences(experimentUtilities.constructBrAPIExternalReferences(program, referenceSource, trialId,null, studyId, obsUnitId, observationId)); + observation.setExternalReferences(experimentUtilities.constructBrAPIExternalReferences(program, referenceSource, trialId, null, studyId, obsUnitId, observationId)); } return observation; } + } From 2384cf7f6033a5c7dbe8cdc92887e4d9ac768487 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 8 Jul 2024 19:33:48 -0400 Subject: [PATCH 187/203] parameterize use of raw types --- .../controllers/ImportController.java | 14 ++++++------ .../controllers/UploadController.java | 10 ++++----- .../appendoverwrite/factory/BrAPIState.java | 2 +- .../factory/action/BrAPIAction.java | 2 +- .../factory/action/WorkflowCreation.java | 6 ++--- .../action/WorkflowReadInitialization.java | 6 ++--- .../factory/action/WorkflowUpdate.java | 8 +++---- .../factory/entity/PendingLocation.java | 9 +++----- .../factory/entity/PendingObservation.java | 3 +-- .../factory/entity/PendingStudy.java | 2 +- .../middleware/commit/BrAPIDatasetCommit.java | 19 ++++++++-------- .../commit/BrAPIObservationCommit.java | 22 +++++++++---------- .../commit/BrAPIObservationUnitCommit.java | 10 ++++----- .../middleware/commit/BrAPIStudyCommit.java | 10 ++++----- .../middleware/commit/BrAPITrialCommit.java | 19 ++++++++-------- .../middleware/commit/LocationCommit.java | 5 ++--- 16 files changed, 70 insertions(+), 77 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index 1a16dadd7..c9ea3ec39 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -77,7 +77,7 @@ public HttpResponse>> getImportTypes Pagination pagination = new Pagination(configs.size(), 1, 1, 0); Metadata metadata = new Metadata(pagination, metadataStatus); - Response> response = new Response(metadata, new DataResponse<>(configs)); + Response> response = new Response<>(metadata, new DataResponse<>(configs)); return HttpResponse.ok(response); } @@ -96,7 +96,7 @@ public HttpResponse>> getMappings(@PathVari Pagination pagination = new Pagination(result.size(), 1, 1, 0); Metadata metadata = new Metadata(pagination, metadataStatus); - Response> response = new Response(metadata, new DataResponse<>(result)); + Response> response = new Response<>(metadata, new DataResponse<>(result)); return HttpResponse.ok(response); } catch (DoesNotExistException e) { log.info(e.getMessage()); @@ -117,7 +117,7 @@ public HttpResponse> createMapping(@PathVariable UUID pr try { AuthenticatedUser actingUser = securityService.getUser(); ImportMapping result = fileImportService.createMapping(programId, actingUser, file); - Response response = new Response(result); + Response response = new Response<>(result); return HttpResponse.ok(response); } catch (DoesNotExistException e) { log.info(e.getMessage()); @@ -141,7 +141,7 @@ public HttpResponse> editMappingFile(@PathVariable UUID try { AuthenticatedUser actingUser = securityService.getUser(); ImportMapping result = fileImportService.updateMappingFile(programId, mappingId, actingUser, file); - Response response = new Response(result); + Response response = new Response<>(result); return HttpResponse.ok(response); } catch (DoesNotExistException e) { log.info(e.getMessage()); @@ -166,7 +166,7 @@ public HttpResponse> editMapping(@PathVariable UUID prog try { AuthenticatedUser actingUser = securityService.getUser(); ImportMapping result = fileImportService.updateMapping(programId, actingUser, mappingId, mapping, validate); - Response response = new Response(result); + Response response = new Response<>(result); return HttpResponse.ok(response); } catch (DoesNotExistException e) { log.error(e.getMessage(), e); @@ -206,7 +206,7 @@ public HttpResponse>> getSystemMappings(@Nu Pagination pagination = new Pagination(result.size(), result.size(), 1, 0); Metadata metadata = new Metadata(pagination, metadataStatus); - Response> response = new Response(metadata, new DataResponse<>(result)); + Response> response = new Response<>(metadata, new DataResponse<>(result)); return HttpResponse.ok(response); } @@ -229,7 +229,7 @@ public HttpResponse>> getWorkflowsForSyste Pagination pagination = new Pagination(workflows.size(), workflows.size(), 1, 0); Metadata metadata = new Metadata(pagination, metadataStatus); - Response> response = new Response(metadata, new DataResponse<>(workflows)); + Response> response = new Response<>(metadata, new DataResponse<>(workflows)); return HttpResponse.ok(response); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java index 5ee6eb062..74928dc10 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java @@ -62,7 +62,7 @@ public HttpResponse> uploadData(@PathVariable UUID prog try { AuthenticatedUser actingUser = securityService.getUser(); ImportResponse result = fileImportService.uploadData(programId, mappingId, actingUser, file); - Response response = new Response(result); + Response response = new Response<>(result); return HttpResponse.ok(response); } catch (DoesNotExistException e) { log.error(e.getMessage(), e); @@ -94,7 +94,7 @@ public HttpResponse> getUploadData(@PathVariable UUID p try { AuthenticatedUser actingUser = securityService.getUser(); Pair result = fileImportService.getDataUpload(uploadId, mapping); - Response response = new Response(result.getRight()); + Response response = new Response<>(result.getRight()); if (result.getLeft().equals(HttpStatus.ACCEPTED)) { return HttpResponse.ok(response).status(result.getLeft()); } else { @@ -141,7 +141,7 @@ public HttpResponse> previewData(@PathVariable UUID pro try { AuthenticatedUser actingUser = securityService.getUser(); ImportResponse result = fileImportService.updateUpload(programId, uploadId, null, actingUser, null, false); - Response response = new Response(result); + Response response = new Response<>(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { log.error(e.getMessage(), e); @@ -167,7 +167,7 @@ public HttpResponse> previewData(@PathVariable UUID pro try { AuthenticatedUser actingUser = securityService.getUser(); ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflow, actingUser, null, false); - Response response = new Response(result); + Response response = new Response<>(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { log.error(e.getMessage(), e); @@ -194,7 +194,7 @@ public HttpResponse> commitData(@PathVariable UUID prog try { AuthenticatedUser actingUser = securityService.getUser(); ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflow, actingUser, userInput, true); - Response response = new Response(result); + Response response = new Response<>(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { log.error(e.getMessage(), e); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java index c9ff1a1c0..2fb63fa01 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java @@ -17,5 +17,5 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; -public interface BrAPIState { +public interface BrAPIState { } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java index 96bcf845a..830de5828 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java @@ -42,7 +42,7 @@ public interface BrAPIAction { * @return An Optional containing the relevant BrAPI state after executing the action. * @throws ApiException if an error occurs during the execution of the action. */ - Optional execute() throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException; + Optional> execute() throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException; /** * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java index 6b1e7cd28..c5f58b852 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java @@ -39,7 +39,7 @@ public class WorkflowCreation implements BrAPIAction { private final ExperimentImportEntity entity; - protected WorkflowCreation(ExperimentImportEntity entity) { + protected WorkflowCreation(ExperimentImportEntity entity) { this.entity = entity; } @@ -50,7 +50,7 @@ protected WorkflowCreation(ExperimentImportEntity entity) { * @return an Optional containing the BrAPI state after execution * @throws ApiException if an error occurs during execution */ - public Optional execute() throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + public Optional> execute() throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { List newMembers = entity.copyWorkflowMembers(ImportObjectState.NEW); try { List createdMembers = entity.brapiPost(newMembers); @@ -77,7 +77,7 @@ public ExperimentImportEntity getEntity() { * @param the type of entity */ @Getter - public class BrAPICreationState implements BrAPIState { + public class BrAPICreationState implements BrAPIState { private final List members; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java index 31e953df6..717a78f97 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java @@ -33,7 +33,7 @@ public class WorkflowReadInitialization implements BrAPIAction { private final ExperimentImportEntity entity; // The entity used for read operations initialization - protected WorkflowReadInitialization(ExperimentImportEntity entity) { + protected WorkflowReadInitialization(ExperimentImportEntity entity) { this.entity = entity; } @@ -44,7 +44,7 @@ protected WorkflowReadInitialization(ExperimentImportEntity entity) { * @return an Optional containing the BrAPIState representing the completed read workflow * @throws ApiException if an error occurs during execution */ - public Optional execute() throws ApiException { + public Optional> execute() throws ApiException { try { List fetchedMembers = entity.brapiRead(); entity.initializeWorkflow(fetchedMembers); @@ -71,7 +71,7 @@ public ExperimentImportEntity getEntity() { * @param the type of entity members contained in the state */ @Getter - public static class BrAPIReadState implements BrAPIState { + public static class BrAPIReadState implements BrAPIState { private final List members; // The list of members fetched during the read operation diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java index ae13ba303..286c9f26c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java @@ -35,12 +35,12 @@ public class WorkflowUpdate implements BrAPIAction { private final ExperimentImportEntity entity; - protected WorkflowUpdate(ExperimentImportEntity entity) { + protected WorkflowUpdate(ExperimentImportEntity entity) { this.entity = entity; } - public Optional execute() throws ApiException { + public Optional> execute() throws ApiException { return saveAndUpdateCache(entity.copyWorkflowMembers(ImportObjectState.MUTATED)); } @@ -64,7 +64,7 @@ public Optional> getBrAPIState() { } } - protected Optional saveAndUpdateCache(List members) throws IllegalArgumentException, ApiException { + protected Optional> saveAndUpdateCache(List members) throws IllegalArgumentException, ApiException { if (members == null) { throw new IllegalArgumentException("BrAPI entity cannot be null"); @@ -76,7 +76,7 @@ protected Optional saveAndUpdateCache(List members) throws Il @Getter - public class BrAPIUpdateState implements BrAPIState { + public class BrAPIUpdateState implements BrAPIState { private final List members; public BrAPIUpdateState(List existingMembers) { this.members = existingMembers; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java index 538b0a7c3..516873f22 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java @@ -62,10 +62,9 @@ public PendingLocation(AppendOverwriteMiddlewareContext context, * * @param members List of entities to be created * @return List of created entities - * @throws ApiException if there is an issue with the API call */ @Override - public List brapiPost(List members) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + public List brapiPost(List members) throws MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { // Construct requests List locationRequests = members.stream() .map(location -> ProgramLocationRequest.builder() @@ -99,11 +98,10 @@ public List brapiRead() throws ApiException { * * @param members List of entities to be updated * @return List of updated entities - * @throws ApiException if there is an issue with the API call * @throws IllegalArgumentException if method arguments are invalid */ @Override - public List brapiPut(List members) throws ApiException, IllegalArgumentException { + public List brapiPut(List members) throws IllegalArgumentException { return new ArrayList<>(); } @@ -112,10 +110,9 @@ public List brapiPut(List members) throws ApiException, IllegalArgumen * * @param members List of entities to be deleted * @return true if deletion is successful, false otherwise - * @throws ApiException if there is an issue with the API call */ @Override - public boolean brapiDelete(List members) throws ApiException { + public boolean brapiDelete(List members) { // TODO: implement delete for program locations return false; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java index bf15734e1..edf63408c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java @@ -94,10 +94,9 @@ public List brapiPost(List members) throws A * Fetch objects required by the workflow from the BrAPI service. * * @return List of fetched entities - * @throws ApiException if there is an issue with the API call */ @Override - public List brapiRead() throws ApiException { + public List brapiRead() { return new ArrayList<>(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java index 4397d205c..546ffff73 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java @@ -186,7 +186,7 @@ public void initializeWorkflow(List members) { .collect(Collectors.toMap(Map.Entry::getKey, e -> Optional.ofNullable(e.getValue().getBrAPIObject().getStudyName()) .map(studyNameScoped -> Utilities.removeProgramKeyAndUnknownAdditionalData(studyNameScoped, importContext.getProgram().getKey())) - .map(nameNoScope -> pendingStudyByNameNoScope.get(nameNoScope)) + .map(pendingStudyByNameNoScope::get) .orElseThrow(() -> new IllegalStateException("Observation unit missing study name: " + e.getKey())))); // Add the maps to the context for use in processing import diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index 15c17145c..7e46b1082 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -41,11 +41,8 @@ public class BrAPIDatasetCommit extends AppendOverwriteMiddleware { private final BrAPICreationFactory brAPICreationFactory; private final BrAPIUpdateFactory brAPIUpdateFactory; - private WorkflowCreation datasetCreation; - private WorkflowUpdate datasetUpdate; - private Optional createdDatasets; - private Optional priorDatasets; - private Optional updatedDatasets; + private Optional.BrAPICreationState> createdDatasets; + private Optional.BrAPIUpdateState> priorDatasets; @Inject public BrAPIDatasetCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUpdateFactory brAPIUpdateFactory) { @@ -56,13 +53,15 @@ public BrAPIDatasetCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUpdate public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { try { - datasetCreation = brAPICreationFactory.datasetWorkflowCreationBean(context); + WorkflowCreation datasetCreation = brAPICreationFactory.datasetWorkflowCreationBean(context); + log.info("creating new datasets in the BrAPI service"); - createdDatasets = datasetCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); - datasetUpdate = brAPIUpdateFactory.datasetWorkflowUpdateBean(context); - priorDatasets = datasetUpdate.getBrAPIState().map(d -> d); + createdDatasets = datasetCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); + WorkflowUpdate datasetUpdate = brAPIUpdateFactory.datasetWorkflowUpdateBean(context); + priorDatasets = datasetUpdate.getBrAPIState(); + log.info("adding new observation variables to datasets"); - updatedDatasets = datasetUpdate.execute().map(d -> (WorkflowUpdate.BrAPIUpdateState) d); + datasetUpdate.execute().map(d -> (WorkflowUpdate.BrAPIUpdateState) d); } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java index b0ce7e696..0b7b6fbfb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -21,9 +21,10 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; @@ -41,11 +42,8 @@ public class BrAPIObservationCommit extends AppendOverwriteMiddleware { private final BrAPICreationFactory brAPICreationFactory; private final BrAPIUpdateFactory brAPIUpdateFactory; - private WorkflowCreation brAPIObservationCreation; - private WorkflowUpdate brAPIObservationUpdate; - private Optional createdBrAPIObservations; - private Optional priorBrAPIObservations; - private Optional updatedObservations; + private Optional.BrAPICreationState> createdBrAPIObservations; + private Optional.BrAPIUpdateState> priorBrAPIObservations; @Inject public BrAPIObservationCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUpdateFactory brAPIUpdateFactory) { @@ -55,13 +53,15 @@ public BrAPIObservationCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUp @Override public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { try { - brAPIObservationCreation = brAPICreationFactory.observationWorkflowCreationBean(context); + WorkflowCreation brAPIObservationCreation = brAPICreationFactory.observationWorkflowCreationBean(context); + log.info("creating new observations in the BrAPI service"); - createdBrAPIObservations = brAPIObservationCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); - brAPIObservationUpdate = brAPIUpdateFactory.observationWorkflowUpdateBean(context); - priorBrAPIObservations = brAPIObservationUpdate.getBrAPIState().map(s -> s); + createdBrAPIObservations = brAPIObservationCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); + WorkflowUpdate brAPIObservationUpdate = brAPIUpdateFactory.observationWorkflowUpdateBean(context); + priorBrAPIObservations = brAPIObservationUpdate.getBrAPIState(); + log.info("updating existing observations in the BrAPI service"); - updatedObservations = brAPIObservationUpdate.execute().map(s -> (WorkflowUpdate.BrAPIUpdateState) s); + brAPIObservationUpdate.execute().map(s -> (WorkflowUpdate.BrAPIUpdateState) s); } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java index 257f03d06..4610be387 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -36,9 +36,8 @@ @Slf4j @Prototype public class BrAPIObservationUnitCommit extends AppendOverwriteMiddleware { - private BrAPICreationFactory brAPICreationFactory; - private WorkflowCreation brAPIObservationUnitCreation; - private Optional createdBrAPIObservationUnits; + private final BrAPICreationFactory brAPICreationFactory; + private Optional.BrAPICreationState> createdBrAPIObservationUnits; @Inject public BrAPIObservationUnitCommit(BrAPICreationFactory brAPICreationFactory) { @@ -47,9 +46,10 @@ public BrAPIObservationUnitCommit(BrAPICreationFactory brAPICreationFactory) { @Override public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { try{ - brAPIObservationUnitCreation = brAPICreationFactory.observationUnitWorkflowCreationBean(context); + WorkflowCreation brAPIObservationUnitCreation = brAPICreationFactory.observationUnitWorkflowCreationBean(context); + log.info("creating new observation units in the BrAPI service"); - createdBrAPIObservationUnits = brAPIObservationUnitCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); + createdBrAPIObservationUnits = brAPIObservationUnitCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java index 8928d5ca6..f10a0b8b6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -36,9 +36,8 @@ @Slf4j @Prototype public class BrAPIStudyCommit extends AppendOverwriteMiddleware { - private BrAPICreationFactory brAPICreationFactory; - private WorkflowCreation brAPIStudyCreation; - private Optional createdBrAPIStudies; + private final BrAPICreationFactory brAPICreationFactory; + private Optional.BrAPICreationState> createdBrAPIStudies; @Inject public BrAPIStudyCommit(BrAPICreationFactory brAPICreationFactory) { @@ -47,9 +46,10 @@ public BrAPIStudyCommit(BrAPICreationFactory brAPICreationFactory) { @Override public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { try { - brAPIStudyCreation = brAPICreationFactory.studyWorkflowCreationBean(context); + WorkflowCreation brAPIStudyCreation = brAPICreationFactory.studyWorkflowCreationBean(context); + log.info("creating new studies in the BrAPI service"); - createdBrAPIStudies = brAPIStudyCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); + createdBrAPIStudies = brAPIStudyCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index 48fafcf37..dbad7b36f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -41,11 +41,8 @@ public class BrAPITrialCommit extends AppendOverwriteMiddleware { private final BrAPICreationFactory brAPICreationFactory; private final BrAPIUpdateFactory brAPIUpdateFactory; - private WorkflowCreation brAPITrialCreation; - private WorkflowUpdate brAPITrialUpdate; - private Optional createdBrAPITrials; - private Optional priorBrAPITrials; - private Optional updatedTrials; + private Optional.BrAPICreationState> createdBrAPITrials; + private Optional.BrAPIUpdateState> priorBrAPITrials; @Inject public BrAPITrialCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUpdateFactory brAPIUpdateFactory) { @@ -55,13 +52,15 @@ public BrAPITrialCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUpdateFa @Override public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { try { - brAPITrialCreation = brAPICreationFactory.trialWorkflowCreationBean(context); + WorkflowCreation brAPITrialCreation = brAPICreationFactory.trialWorkflowCreationBean(context); + log.info("creating new trials in the BrAPI service"); - createdBrAPITrials = brAPITrialCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); - brAPITrialUpdate = brAPIUpdateFactory.trialWorkflowUpdateBean(context); - priorBrAPITrials = brAPITrialUpdate.getBrAPIState().map(s -> s); + createdBrAPITrials = brAPITrialCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); + WorkflowUpdate brAPITrialUpdate = brAPIUpdateFactory.trialWorkflowUpdateBean(context); + priorBrAPITrials = brAPITrialUpdate.getBrAPIState(); + log.info("updating existing trials in the BrAPI service"); - updatedTrials = brAPITrialUpdate.execute().map(s -> (WorkflowUpdate.BrAPIUpdateState) s); + brAPITrialUpdate.execute().map(s -> (WorkflowUpdate.BrAPIUpdateState) s); } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java index 0697a4bbc..34fda1fca 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -36,8 +36,7 @@ @Slf4j @Prototype public class LocationCommit extends AppendOverwriteMiddleware { - private BrAPICreationFactory brAPICreationFactory; - private WorkflowCreation locationCreation; + private final BrAPICreationFactory brAPICreationFactory; private Optional createdLocations; @Inject @@ -47,7 +46,7 @@ public LocationCommit(BrAPICreationFactory brAPICreationFactory) { @Override public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { try { - locationCreation = brAPICreationFactory.locationWorkflowCreationBean(context); + WorkflowCreation locationCreation = brAPICreationFactory.locationWorkflowCreationBean(context); log.info("creating new locationss in the Deltabreed database"); createdLocations = locationCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { From c37f91c009962c4da575da06a299ecd7d73e9982 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 8 Jul 2024 20:18:54 -0400 Subject: [PATCH 188/203] make WorkflowUpdate creation private to factory --- .../factory/action/BrAPIUpdateFactory.java | 79 +++++++++++++++++-- .../middleware/commit/BrAPIDatasetCommit.java | 4 +- .../commit/BrAPIObservationCommit.java | 4 +- .../middleware/commit/BrAPITrialCommit.java | 4 +- 4 files changed, 79 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java index 9c361d746..cb73627b1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java @@ -20,13 +20,22 @@ import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.PendingEntityFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import javax.inject.Inject; +import java.util.List; +import java.util.Optional; @Factory public class BrAPIUpdateFactory { @@ -37,19 +46,19 @@ public BrAPIUpdateFactory(PendingEntityFactory pendingEntityFactory) { this.pendingEntityFactory = pendingEntityFactory; } - public static WorkflowUpdate trialWorkflowUpdate(AppendOverwriteMiddlewareContext context, + private static WorkflowUpdate trialWorkflowUpdate(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { - return new WorkflowUpdate(pendingEntityFactory.pendingTrialBean(context)); + return new WorkflowUpdate<>(pendingEntityFactory.pendingTrialBean(context)); } - public static WorkflowUpdate observationWorkflowUpdate(AppendOverwriteMiddlewareContext context, + private static WorkflowUpdate observationWorkflowUpdate(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { - return new WorkflowUpdate(pendingEntityFactory.pendingObservationBean(context)); + return new WorkflowUpdate<>(pendingEntityFactory.pendingObservationBean(context)); } - public static WorkflowUpdate datasetWorkflowUpdate(AppendOverwriteMiddlewareContext context, + private static WorkflowUpdate datasetWorkflowUpdate(AppendOverwriteMiddlewareContext context, PendingEntityFactory pendingEntityFactory) { - return new WorkflowUpdate(pendingEntityFactory.pendingDatasetBean(context)); + return new WorkflowUpdate<>(pendingEntityFactory.pendingDatasetBean(context)); } @Bean @@ -69,4 +78,62 @@ public WorkflowUpdate observationWorkflowUpdateBean(AppendOver public WorkflowUpdate datasetWorkflowUpdateBean(AppendOverwriteMiddlewareContext context) { return datasetWorkflowUpdate(context, pendingEntityFactory); } + + @Slf4j + @Prototype + public static class WorkflowUpdate implements BrAPIAction { + private final ExperimentImportEntity entity; + + private WorkflowUpdate(ExperimentImportEntity entity) { + + this.entity = entity; + } + + public Optional> execute() throws ApiException { + return saveAndUpdateCache(entity.copyWorkflowMembers(ImportObjectState.MUTATED)); + } + + /** + * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. + * + * @return The ExperimentImportEntity representing the BrAPI entity being acted on. + */ + @Override + public ExperimentImportEntity getEntity() { + return null; + } + + public Optional> getBrAPIState() { + try { + return Optional.of(new BrAPIUpdateState(entity.getBrAPIState(ImportObjectState.MUTATED))); + } catch (ApiException e) { + // TODO: add specific error messages to entity service + log.error("Error getting..."); + throw new InternalServerException("Error getting...", e); + } + + } + protected Optional> saveAndUpdateCache(List members) throws IllegalArgumentException, ApiException { + + if (members == null) { + throw new IllegalArgumentException("BrAPI entity cannot be null"); + } + List savedMembers = entity.brapiPut(members); + entity.updateWorkflow(savedMembers); + return Optional.of(new BrAPIUpdateState(savedMembers)); + } + + + @Getter + public class BrAPIUpdateState implements BrAPIState { + private final List members; + + public BrAPIUpdateState(List existingMembers) { this.members = existingMembers; } + + public boolean restore() throws ApiException { + return saveAndUpdateCache(this.getMembers()).isPresent(); + } + } + + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java index 7e46b1082..368673f19 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -24,7 +24,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; @@ -42,7 +42,7 @@ public class BrAPIDatasetCommit extends AppendOverwriteMiddleware { private final BrAPICreationFactory brAPICreationFactory; private final BrAPIUpdateFactory brAPIUpdateFactory; private Optional.BrAPICreationState> createdDatasets; - private Optional.BrAPIUpdateState> priorDatasets; + private Optional.BrAPIUpdateState> priorDatasets; @Inject public BrAPIDatasetCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUpdateFactory brAPIUpdateFactory) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java index 0b7b6fbfb..779d75493 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -25,7 +25,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; @@ -43,7 +43,7 @@ public class BrAPIObservationCommit extends AppendOverwriteMiddleware { private final BrAPICreationFactory brAPICreationFactory; private final BrAPIUpdateFactory brAPIUpdateFactory; private Optional.BrAPICreationState> createdBrAPIObservations; - private Optional.BrAPIUpdateState> priorBrAPIObservations; + private Optional.BrAPIUpdateState> priorBrAPIObservations; @Inject public BrAPIObservationCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUpdateFactory brAPIUpdateFactory) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java index dbad7b36f..3ca112b21 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -24,7 +24,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory.WorkflowUpdate; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; @@ -42,7 +42,7 @@ public class BrAPITrialCommit extends AppendOverwriteMiddleware { private final BrAPICreationFactory brAPICreationFactory; private final BrAPIUpdateFactory brAPIUpdateFactory; private Optional.BrAPICreationState> createdBrAPITrials; - private Optional.BrAPIUpdateState> priorBrAPITrials; + private Optional.BrAPIUpdateState> priorBrAPITrials; @Inject public BrAPITrialCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUpdateFactory brAPIUpdateFactory) { From 2f93738d1a14db8cf2dee664dcc5129385feb776 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 8 Jul 2024 20:22:49 -0400 Subject: [PATCH 189/203] delete unused WorkflowUpdate class --- .../factory/action/WorkflowUpdate.java | 89 ------------------- 1 file changed, 89 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java deleted file mode 100644 index 286c9f26c..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowUpdate.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; - -import io.micronaut.context.annotation.Prototype; -import io.micronaut.http.server.exceptions.InternalServerException; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity; -import org.checkerframework.checker.units.qual.A; - -import java.util.List; -import java.util.Optional; - -@Slf4j -@Prototype -public class WorkflowUpdate implements BrAPIAction { - private final ExperimentImportEntity entity; - - protected WorkflowUpdate(ExperimentImportEntity entity) { - - this.entity = entity; - } - - public Optional> execute() throws ApiException { - return saveAndUpdateCache(entity.copyWorkflowMembers(ImportObjectState.MUTATED)); - } - - /** - * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. - * - * @return The ExperimentImportEntity representing the BrAPI entity being acted on. - */ - @Override - public ExperimentImportEntity getEntity() { - return null; - } - - public Optional> getBrAPIState() { - try { - return Optional.of(new BrAPIUpdateState(entity.getBrAPIState(ImportObjectState.MUTATED))); - } catch (ApiException e) { - // TODO: add specific error messages to entity service - log.error("Error getting..."); - throw new InternalServerException("Error getting...", e); - } - - } - protected Optional> saveAndUpdateCache(List members) throws IllegalArgumentException, ApiException { - - if (members == null) { - throw new IllegalArgumentException("BrAPI entity cannot be null"); - } - List savedMembers = entity.brapiPut(members); - entity.updateWorkflow(savedMembers); - return Optional.of(new BrAPIUpdateState(savedMembers)); - } - - - @Getter - public class BrAPIUpdateState implements BrAPIState { - private final List members; - - public BrAPIUpdateState(List existingMembers) { this.members = existingMembers; } - - public boolean restore() throws ApiException { - return saveAndUpdateCache(this.getMembers()).isPresent(); - } - } - -} From ab632bc4cc290148de44bb830e09631491b7e49f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 8 Jul 2024 21:07:29 -0400 Subject: [PATCH 190/203] replace types keyword with old type syntax in maven build action --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index da9e70b3f..4263a6251 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ name: maven build on: pull_request: - types: [opened, edited] + type: [opened, edited, synchronize] jobs: build: From f17c1ce7af3062447d89291eeea217146ac86235 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 8 Jul 2024 21:09:58 -0400 Subject: [PATCH 191/203] revert keyword with synchronize type --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4263a6251..e2244461f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ name: maven build on: pull_request: - type: [opened, edited, synchronize] + types: [opened, edited, synchronize] jobs: build: From e74d653877cf861923316d758534fcbd5ce75e66 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 8 Jul 2024 21:19:06 -0400 Subject: [PATCH 192/203] trigger build --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e2244461f..eaccb76a0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,9 +17,10 @@ name: maven build + on: pull_request: - types: [opened, edited, synchronize] + type: [opened, edited] jobs: build: From 40446a9a78e71021e2b9935ee25a7da1e997735c Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 8 Jul 2024 21:31:48 -0400 Subject: [PATCH 193/203] specify branch --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eaccb76a0..43d6c04da 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,9 @@ name: maven build on: pull_request: - type: [opened, edited] + types: [opened, synchronize, edited] + branches: + - 'feature/**' jobs: build: From f7585ee717414247f9483626a0f2db2af515428d Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 8 Jul 2024 21:33:52 -0400 Subject: [PATCH 194/203] revert --- .github/workflows/build.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 43d6c04da..eaccb76a0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,9 +20,7 @@ name: maven build on: pull_request: - types: [opened, synchronize, edited] - branches: - - 'feature/**' + type: [opened, edited] jobs: build: From 606224a95f84ea3e3d454c852036eecd53cb087c Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 8 Jul 2024 21:36:31 -0400 Subject: [PATCH 195/203] revert --- .github/workflows/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eaccb76a0..0f77a41e0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,6 @@ name: maven build - on: pull_request: type: [opened, edited] @@ -60,4 +59,4 @@ jobs: JWT_SECRET: ${{ secrets.JWT_SECRET }} OAUTH_CLIENT_ID: 123abc OAUTH_CLIENT_SECRET: asdfljkhalkbaldsfjasdfi238497098asdf - BRAPI_REFERENCE_SOURCE: breedinginsight.org + BRAPI_REFERENCE_SOURCE: breedinginsight.org \ No newline at end of file From a0b644f2a495f5f0b6556dd0956924e908461f9b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 10 Jul 2024 10:12:03 -0400 Subject: [PATCH 196/203] fix test --- .../brapps/importer/ExperimentFileImportTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index f66453fc7..ed743f72d 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1230,7 +1230,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { Map requestBody = new HashMap<>(); requestBody.put("overwrite", "true"); requestBody.put("overwriteReason", "testing"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), requestBody, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); From f2c55c60b7c0b345534b81306095367dd9b1fcb5 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 10 Jul 2024 10:36:37 -0400 Subject: [PATCH 197/203] disable tests that need cache mocking fixed --- .../TraitControllerIntegrationTest.java | 68 ++++++++++--------- .../BrAPIServiceFilterIntegrationTest.java | 6 +- 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java index 9d908a9e3..ebfeec7ed 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java @@ -232,6 +232,7 @@ public Species getTestSpecies() { return species.get(0); } + @Disabled("Temporarily disabled, see BI-2238") @Test @SneakyThrows @Order(1) @@ -245,6 +246,7 @@ public void getTraitsNoExistInBrAPI() { assertEquals("Could not find trait in returned brapi server results", fetchException.get().get().getMessage()); } + @Disabled("Temporarily disabled, see BI-2238") @Test @SneakyThrows @Order(1) @@ -263,7 +265,7 @@ public void getTraitTagsDefaultFavorites() { assertEquals(1, data.size(), "Wrong number of results returned."); assertEquals("favorites", data.get(0).getAsString(), "Wrong default tag returned"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @SneakyThrows @Order(2) @@ -284,6 +286,7 @@ public void getTraitSingleNoExistInBrAPI() { } //region POST traits + @Test @SneakyThrows @Order(2) @@ -314,7 +317,7 @@ public void createTraitsBrAPIError() { assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, e.getStatus()); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(3) public void postTraitsMultiple() { @@ -399,7 +402,7 @@ public void postTraitsMultiple() { validTraits = traits; } - + @Disabled("Temporarily disabled, see BI-2238") @Test @SneakyThrows @Order(4) @@ -494,7 +497,7 @@ public void checkBrapiValues() { variable.getTrait().getTraitName(), "Unexpected trait name"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @SneakyThrows @Order(4) @@ -587,7 +590,7 @@ private BrAPIObservationVariable getBrapiVariable(String variableName, String pr return searchResult.orElseThrow(); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @SneakyThrows @Order(4) @@ -612,7 +615,7 @@ public void getTraitTags() { assertTrue(tagsList.equals(foundTags), "Returned tags do not equal expected tags"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(4) public void postTraitsMultipleExistInDb() { @@ -639,7 +642,7 @@ public void postTraitsMultipleExistInDb() { JsonObject duplicateError = errors.get(0).getAsJsonObject(); assertEquals(409, duplicateError.get("httpStatusCode").getAsInt(), "Wrong error code returned"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(4) public void postTraitsNotSharedBetweenPrograms() { @@ -663,7 +666,7 @@ public void postTraitsNotSharedBetweenPrograms() { assertEquals(validTraits.size(), data.size(), "Traits were ignored, but should not be"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(4) public void createTraitsDuplicateInRequest() { @@ -690,7 +693,7 @@ public void createTraitsDuplicateInRequest() { JsonObject duplicateError = errors.get(0).getAsJsonObject(); assertEquals(409, duplicateError.get("httpStatusCode").getAsInt(), "Wrong error code returned"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(4) public void createTraitsLevelDoesNotExist() { @@ -721,7 +724,7 @@ public void createTraitsLevelDoesNotExist() { JsonArray data = result.getAsJsonArray("data"); assertEquals(1, data.size(), "Wrong number of traits"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(4) public void createTraitsValidationError() { @@ -762,7 +765,7 @@ public void createTraitsValidationError() { assertTrue(error.has("httpStatus"), "httpStatus field not included in return"); assertTrue(error.has("httpStatusCode"), "httpStatusCode field not included in return"); } - + @Disabled("Temporarily disabled, see BI-2238") public void setBrAPIProperties(Trait trait) { // Trait @@ -787,6 +790,7 @@ public void setBrAPIProperties(Trait trait) { //endregion //region GET Traits + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(5) public void getTraitsSuccess() { @@ -819,7 +823,7 @@ public void getTraitsSuccess() { throw new AssertionFailedError("Both traits were not returned"); } } - + @Disabled("Temporarily disabled, see BI-2238") @Test @SneakyThrows @Order(5) @@ -881,7 +885,7 @@ public void getTraitsBadFullQueryParam() { }); assertEquals(HttpStatus.BAD_REQUEST, e.getStatus()); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @SneakyThrows @Order(5) @@ -898,7 +902,7 @@ public void getTraitSingleSuccess() { checkTraitFullResponse(result, validTraits.get(0)); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(5) public void getTraitSingleNoExist() { @@ -912,7 +916,7 @@ public void getTraitSingleNoExist() { }); assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(5) public void getTraitSingleProgramNotExist() { @@ -941,7 +945,7 @@ public void editableTraitNotExist() { }); assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(5) public void editableProgramNotExist() { @@ -956,7 +960,7 @@ public void editableProgramNotExist() { }); assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(6) public void editableNoObservations() { @@ -971,7 +975,7 @@ public void editableNoObservations() { JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(true, result.getAsJsonPrimitive("editable").getAsBoolean(), "expected to be editable"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(7) @SneakyThrows @@ -1003,7 +1007,7 @@ public void editableHasObservations() { assertEquals(false, result.getAsJsonPrimitive("editable").getAsBoolean(), "expected not to be editable"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(8) public void editableNotAllowed() { @@ -1030,7 +1034,7 @@ public void editableNotAllowed() { assertEquals(HttpStatus.METHOD_NOT_ALLOWED, e.getStatus()); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @SneakyThrows @Order(9) @@ -1122,7 +1126,7 @@ public void checkTraitFullResponse(JsonObject traitJson, Trait trait) { } } //endregion - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(10) public void postTraitNominalMissingCategoryValue() { @@ -1178,7 +1182,7 @@ public void postTraitNominalMissingCategoryValue() { assertTrue(categoryError.has("httpStatus"), "httpStatus field not included in return"); assertTrue(categoryError.has("httpStatusCode"), "httpStatusCode field not included in return"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(10) public void postTraitOrdinalMissingCategoryVariables() { @@ -1246,7 +1250,7 @@ public void postTraitOrdinalMissingCategoryVariables() { JsonObject labelError = secondCategoryError.getAsJsonArray("errors").get(0).getAsJsonObject(); assertEquals("scale.categories.label", labelError.get("field").getAsString(), "wrong error returned"); } - + @Disabled("Temporarily disabled, see BI-2238") @SneakyThrows @Test @Order(11) @@ -1302,7 +1306,7 @@ public void getTraitsQuery() { assertEquals(allTraits.size(), pagination.get("totalCount").getAsInt(), "Wrong total count"); assertEquals(1, pagination.get("currentPage").getAsInt(), "Wrong current page"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(12) public void searchTraits() { @@ -1325,7 +1329,7 @@ public void searchTraits() { assertEquals(11, data.size(), "Wrong page size"); TestUtils.checkStringSorting(data, "observationVariableName", SortOrder.ASC); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(12) public void searchTraitsByDescription() { @@ -1347,7 +1351,7 @@ public void searchTraitsByDescription() { assertEquals(11, data.size(), "Wrong page size"); TestUtils.checkStringSorting(data, "traitDescription", SortOrder.ASC); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(13) public void postTraitComputation() { @@ -1399,7 +1403,7 @@ public void postTraitComputation() { trait1.setId(UUID.fromString(trait.get("id").getAsString())); validTraits.add(trait1); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(14) public void putTraitComputation() { @@ -1447,7 +1451,7 @@ public void putTraitComputation() { assertEquals(updateTrait.getMethod().getMethodClass(), trait.get("method").getAsJsonObject().get("methodClass").getAsString(), "wrong method class"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(14) public void putTraitMultipleValidationErrors() { @@ -1499,7 +1503,7 @@ public void putTraitMultipleValidationErrors() { JsonObject error = errors.get(0).getAsJsonObject(); assertEquals("traitId", error.get("field").getAsString(), "wrong error returned"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(15) public void archiveTrait() { @@ -1517,7 +1521,7 @@ public void archiveTrait() { JsonObject result = JsonParser.parseString(response.getBody().get()).getAsJsonObject().getAsJsonObject("result"); assertFalse(result.get("active").getAsBoolean(), "Trait not archived"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(15) public void restoreTrait() { @@ -1535,7 +1539,7 @@ public void restoreTrait() { JsonObject result = JsonParser.parseString(response.getBody().get()).getAsJsonObject().getAsJsonObject("result"); assertTrue(result.get("active").getAsBoolean(), "Trait not archived"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(16) public void putTraitIdDoesNotExist() { @@ -1575,7 +1579,7 @@ public void putTraitIdDoesNotExist() { JsonObject error = errors.get(0).getAsJsonObject(); assertEquals("traitId", error.get("field").getAsString(), "wrong error returned"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(17) public void archiveTraitIdNotExist() { diff --git a/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java index f4bf5d5ab..bd565374e 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java @@ -208,7 +208,7 @@ public void retrieveTestData() { validProgram = programDao.findAll().get(0); validVariable = traitDao.findAll().get(0); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(1) @SneakyThrows @@ -240,7 +240,7 @@ public void urlChangesForDifferentRequestsCallOne() { // Check that our error is not an assertion error assertEquals(coreUrl, urlCheck.get().get(), "URL was not as expected"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @Order(2) @SneakyThrows @@ -271,7 +271,7 @@ public void urlChangesForDifferentRequestsCallTwo() { assertEquals(coreUrl1, urlCheck.get().get(), "Url was not as expected"); } - + @Disabled("Temporarily disabled, see BI-2238") @Test @SneakyThrows public void getTraitSingleEditableUsesFilter() { From 6fe9775b6c676c8a41f29ec3a735d334c87a0cba Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 10 Jul 2024 12:19:47 -0400 Subject: [PATCH 198/203] update controller tests to use new exp workflow --- .../BrAPIObservationLevelsControllerIntegrationTest.java | 7 +++++-- .../v2/BrAPIObservationUnitControllerIntegrationTest.java | 7 +++++-- .../v2/BrAPIObservationsControllerIntegrationTest.java | 7 +++++-- .../brapi/v2/BrAPIStudiesControllerIntegrationTest.java | 7 +++++-- ...APIV2ObservationVariableControllerIntegrationTest.java | 8 ++++++-- .../brapi/v2/ListControllerIntegrationTest.java | 7 +++++-- 6 files changed, 31 insertions(+), 12 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java index 85f6c6773..1d410ad89 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java @@ -92,6 +92,7 @@ public class BrAPIObservationLevelsControllerIntegrationTest extends BrAPITest { @Client("/${micronaut.bi.api.version}") private RxHttpClient client; + private String newExperimentWorkflowId; private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .create(); @@ -100,6 +101,7 @@ public class BrAPIObservationLevelsControllerIntegrationTest extends BrAPITest { void setup() throws Exception { FannyPack fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); ImportTestUtils importTestUtils = new ImportTestUtils(); + newExperimentWorkflowId = importTestUtils.getExperimentWorkflowId(client, 0); FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); @@ -183,13 +185,14 @@ void setup() throws Exception { rows.add(row2); // Import test experiment, environments, and any observations - JsonObject importResult = importTestUtils.uploadAndFetch( + JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( writeDataToFile(rows, traits), null, true, client, program, - mappingId); + mappingId, + newExperimentWorkflowId); experimentId = importResult .get("preview").getAsJsonObject() .get("rows").getAsJsonArray() diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java index 3a3d8ffa3..737eb55bf 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java @@ -96,6 +96,7 @@ public class BrAPIObservationUnitControllerIntegrationTest extends BrAPITest { @Client("/${micronaut.bi.api.version}") private RxHttpClient client; + private String newExperimentWorkflowId; private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .create(); @@ -104,6 +105,7 @@ public class BrAPIObservationUnitControllerIntegrationTest extends BrAPITest { void setup() throws Exception { FannyPack fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); ImportTestUtils importTestUtils = new ImportTestUtils(); + newExperimentWorkflowId = importTestUtils.getExperimentWorkflowId(client, 0); FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); @@ -185,13 +187,14 @@ void setup() throws Exception { rows.add(row2); // Import test experiment, environments, and any observations - JsonObject importResult = importTestUtils.uploadAndFetch( + JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( writeDataToFile(rows, traits), null, true, client, program, - mappingId); + mappingId, + newExperimentWorkflowId); experimentId = importResult .get("preview").getAsJsonObject() .get("rows").getAsJsonArray() diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java index 16d7261fb..7e1db099f 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java @@ -94,6 +94,7 @@ public class BrAPIObservationsControllerIntegrationTest extends BrAPITest { @Client("/${micronaut.bi.api.version}") private RxHttpClient client; + private String newExperimentWorkflowId; private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .create(); @@ -102,6 +103,7 @@ public class BrAPIObservationsControllerIntegrationTest extends BrAPITest { void setup() throws Exception { FannyPack fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); ImportTestUtils importTestUtils = new ImportTestUtils(); + newExperimentWorkflowId = importTestUtils.getExperimentWorkflowId(client, 0); FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); @@ -183,13 +185,14 @@ void setup() throws Exception { rows.add(row2); // Import test experiment, environments, and any observations - JsonObject importResult = importTestUtils.uploadAndFetch( + JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( writeDataToFile(rows, traits), null, true, client, program, - mappingId); + mappingId, + newExperimentWorkflowId); experimentId = importResult .get("preview").getAsJsonObject() .get("rows").getAsJsonArray() diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java index ac276003c..c95a17a6c 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java @@ -96,6 +96,7 @@ public class BrAPIStudiesControllerIntegrationTest extends BrAPITest { @Client("/${micronaut.bi.api.version}") private RxHttpClient client; + private String newExperimentWorkflowId; private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .create(); @@ -104,6 +105,7 @@ public class BrAPIStudiesControllerIntegrationTest extends BrAPITest { void setup() throws Exception { FannyPack fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); ImportTestUtils importTestUtils = new ImportTestUtils(); + newExperimentWorkflowId = importTestUtils.getExperimentWorkflowId(client, 0); FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); @@ -185,13 +187,14 @@ void setup() throws Exception { rows.add(row2); // Import test experiment, environments, and any observations - JsonObject importResult = importTestUtils.uploadAndFetch( + JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( writeDataToFile(rows, traits), null, true, client, program, - mappingId); + mappingId, + newExperimentWorkflowId); experimentId = importResult .get("preview").getAsJsonObject() .get("rows").getAsJsonArray() diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java index 58c20194f..40665c03c 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java @@ -93,6 +93,8 @@ public class BrAPIV2ObservationVariableControllerIntegrationTest extends BrAPITe @Client("/${micronaut.bi.api.version}") private RxHttpClient client; + private String newExperimentWorkflowId; + private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .create(); @@ -101,6 +103,7 @@ public class BrAPIV2ObservationVariableControllerIntegrationTest extends BrAPITe void setup() throws Exception { FannyPack fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); ImportTestUtils importTestUtils = new ImportTestUtils(); + newExperimentWorkflowId = importTestUtils.getExperimentWorkflowId(client, 0); FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); @@ -184,13 +187,14 @@ void setup() throws Exception { rows.add(row2); // Import test experiment, environments, and any observations - JsonObject importResult = importTestUtils.uploadAndFetch( + JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( writeDataToFile(rows, expTraits), null, true, client, program, - mappingId); + mappingId, + newExperimentWorkflowId); experimentId = importResult .get("preview").getAsJsonObject() .get("rows").getAsJsonArray() diff --git a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java index fdabb53db..9515b72a6 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java @@ -84,10 +84,13 @@ public class ListControllerIntegrationTest extends BrAPITest { @Inject private OntologyService ontologyService; + private String newExperimentWorkflowId; + @BeforeAll @SneakyThrows public void setup() { importTestUtils = new ImportTestUtils(); + newExperimentWorkflowId = importTestUtils.getExperimentWorkflowId(client, 0); Map setupObjects = importTestUtils.setup(client, gson, dsl, speciesService, userDAO, super.getBrapiDsl(), "ExperimentsTemplateMap"); mappingId = (String) setupObjects.get("mappingId"); program = (Program) setupObjects.get("program"); @@ -141,8 +144,8 @@ public void setup() { newExp.put(ExperimentObservation.Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch( - importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId + JsonObject result = importTestUtils.uploadAndFetchWorkflow( + importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId ); } From e97baac5516bcd55dc25c1c28ac9eb2aabeb8cd1 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 10 Jul 2024 12:42:35 -0400 Subject: [PATCH 199/203] update more controller tests --- .../SampleSubmissionControllerIntegrationTest.java | 7 +++++-- .../brapi/v2/ExperimentControllerIntegrationTest.java | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java index fd221e810..ec28c689b 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java @@ -90,11 +90,13 @@ public class SampleSubmissionControllerIntegrationTest extends BrAPITest { @Client("/${micronaut.bi.api.version}") private RxHttpClient client; + private String newExperimentWorkflowId; private final Gson gson = new BrAPIClient().getJSON().getGson(); @BeforeAll void setup() throws Exception { importTestUtils = new ImportTestUtils(); + newExperimentWorkflowId = importTestUtils.getExperimentWorkflowId(client, 0); FannyPack fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); @@ -404,13 +406,14 @@ private String createExperiment(Program program) throws IOException, Interrupted .getAsJsonArray("data") .get(0).getAsJsonObject().get("id").getAsString(); - JsonObject importResult = importTestUtils.uploadAndFetch( + JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null), null, true, client, program, - expMappingId); + expMappingId, + newExperimentWorkflowId); return importResult .get("preview").getAsJsonObject() .get("rows").getAsJsonArray() diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index 630c29c85..9aa6e07a1 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -80,6 +80,7 @@ public class ExperimentControllerIntegrationTest extends BrAPITest { @Client("/${micronaut.bi.api.version}") private RxHttpClient client; + private String newExperimentWorkflowId; private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .create(); @@ -88,6 +89,7 @@ public class ExperimentControllerIntegrationTest extends BrAPITest { void setup() throws Exception { FannyPack fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); ImportTestUtils importTestUtils = new ImportTestUtils(); + newExperimentWorkflowId = importTestUtils.getExperimentWorkflowId(client, 0); FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); @@ -169,13 +171,14 @@ void setup() throws Exception { rows.add(row2); // Import test experiment, environments, and any observations - JsonObject importResult = importTestUtils.uploadAndFetch( + JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( writeDataToFile(rows, traits), null, true, client, program, - mappingId); + mappingId, + newExperimentWorkflowId); experimentId = importResult .get("preview").getAsJsonObject() .get("rows").getAsJsonArray() From 3d215553a6091250a1e435bca7384d23a8f55cfc Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 10 Jul 2024 16:28:43 -0400 Subject: [PATCH 200/203] clean up code --- .../model/imports/DomainImportService.java | 4 - .../model/response/ImportPreviewResponse.java | 1 - .../experiment/ExperimentUtilities.java | 82 ------------------- .../ExperimentWorkflowNavigator.java | 4 +- .../AppendOverwritePhenotypesWorkflow.java | 2 +- .../factory/data/OverwrittenData.java | 1 - .../factory/entity/PendingDataset.java | 5 +- .../factory/entity/PendingGermplasm.java | 9 +- 8 files changed, 8 insertions(+), 100 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index a8dce4090..7187b4492 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -22,11 +22,7 @@ import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.Workflow; -import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; -import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; -import javax.inject.Provider; import javax.inject.Singleton; import java.util.List; import java.util.Optional; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/response/ImportPreviewResponse.java b/src/main/java/org/breedinginsight/brapps/importer/model/response/ImportPreviewResponse.java index 23fec70c6..b46fc6335 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/response/ImportPreviewResponse.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/response/ImportPreviewResponse.java @@ -17,7 +17,6 @@ package org.breedinginsight.brapps.importer.model.response; -import lombok.Builder; import lombok.Getter; import lombok.Setter; import org.breedinginsight.brapps.importer.model.imports.PendingImport; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 090d24d07..a215af40f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -121,44 +121,6 @@ public Optional clone(T obj, Class clazz) { } } - /** - * Retrieves a list of new objects of type T from the provided map of pending import objects by filtering out null previews and objects with state other than NEW, - * mapping the BrAPI object from each preview, cloning it to the specified class type, and collecting the non-empty results into a list. - * - * @param objectsByName a map of pending import objects with V keys and PendingImportObject values - * @param clazz the target class type for cloning the BrAPI object - * @param the type of new objects to be retrieved - * @param the type of keys in the map of pending import objects - * @return a list of cloned new objects of type T extracted from the input map - */ - public List getNewObjects(Map> objectsByName, Class clazz) { - return objectsByName.values().stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW) - .map(PendingImportObject::getBrAPIObject) - .map(b->clone(b, clazz)) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toList()); - } - - /** - * Copies mutated objects from a cache map to a new list. - * Only objects with ImportObjectState MUTATED are included in the copied list. - * - * @param pendingCacheMap a map containing PendingImportObject objects to be copied - * @param clazz a Class object representing the type of objects to be copied - * @return a List of copied objects of type T - */ - public List copyMutationsFromCache(Map> pendingCacheMap, Class clazz) { - return pendingCacheMap.values().stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED) - .map(PendingImportObject::getBrAPIObject) - .map(b -> clone(b, clazz)) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toList()); - } - /** * Copies the pending BrAPI objects from the workflow cache map based on the provided import object status. * @@ -185,35 +147,6 @@ public List copyWorkflowCachePendingBrAPIObjects(Map Type parameter for the objects in the Map. - * @param Type parameter for the keys in the Map. - * @return A Map of String keys (DB IDs) and objects of type T as values based on the filter logic. - * @throws RuntimeException if an exception occurs while applying the DB ID filter to an object. - */ - public Map getMutationsByObjectId(Map> objectsByName, Function dbIdFilter, Class clazz) { - return objectsByName.values().stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED) - .map(PendingImportObject::getBrAPIObject) - .map(b -> clone(b, clazz)) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors - .toMap(brapiObj -> { - try { - return dbIdFilter.apply(brapiObj); - } catch (Exception e) { - throw new RuntimeException(e); - } - }, - brapiObj -> brapiObj)); - } - /** * Converts a list of BrAPI imports to a list of ExperimentObservations. * @@ -256,21 +189,6 @@ public static String createObservationUnitKey(String studyName, String obsUnitNa return studyName + obsUnitName; } - /** - * Returns the single value from the provided map if it contains exactly one entry. - * - * @param map the map from which to extract the single value - * @param message the message to be included in the exception thrown if the map does not contain exactly one entry - * @return the single value from the map - * @throws UnprocessableEntityException if the map does not contain only one entry - */ - public V getSingleEntryValue(Map map, String message) throws UnprocessableEntityException { - if (map.size() != 1) { - throw new UnprocessableEntityException(message); - } - return map.values().iterator().next(); - } - // Module/Script-level documentation /** * This method is used to retrieve the value from a map if the map contains only one entry. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index 520fc53d5..6e1d313f7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -73,7 +73,7 @@ public List getWorkflows() { .map(workflow -> workflow.process(null)) // Process each workflow with a null context .filter(Optional::isPresent) // Filter out any workflows that do not return a result .map(Optional::get) // Extract the result from Optional - .map(result -> result.getWorkflow()) // Retrieve the workflow metadata + .map(ImportWorkflowResult::getWorkflow) // Retrieve the workflow metadata .collect(Collectors.toList()); // Collect the workflow metadata into a list // Set the order field for each workflow based on its position in the list @@ -138,7 +138,7 @@ public String getName() { * @return true if the value is equal to the ID, false otherwise. */ public boolean isEqual(String value) { - return Optional.ofNullable(id.equals(value)).orElse(false); + return value.equals(id); } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index df529af99..cab42d6fc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -156,7 +156,7 @@ public Optional process(ImportServiceContext context) { Optional brapiCommitException = Optional.ofNullable(brapiCommittedContext.getAppendOverwriteWorkflowContext().getProcessError()); if (brapiCommitException.isPresent()) { - log.debug(String.format("%s in %s", brapiCommitException.get().getException().getClass()), brapiCommitException.get().getLocalTransactionName()); + log.debug(String.format("%s in %s", brapiCommitException.get().getException().getClass(), brapiCommitException.get().getLocalTransactionName())); result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.ofNullable(brapiCommitException.get().getException()))); return result; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java index 78bd51a30..51f839494 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java @@ -38,7 +38,6 @@ import javax.inject.Inject; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.List; import java.util.Optional; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java index 3e8dd03e5..a5d3e4ca2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java @@ -66,7 +66,7 @@ public PendingDataset(AppendOverwriteMiddlewareContext context, * @throws ApiException if there is an issue with the API call */ @Override - public List brapiPost(List members) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + public List brapiPost(List members) throws ApiException { // Construct BrAPI list requests List requests = members.stream().map(details -> { BrAPIListNewRequest request = new BrAPIListNewRequest(); @@ -154,10 +154,9 @@ public List brapiPut(List members) throws ApiException, IllegalArgumen * * @param members List of entities to be deleted * @return true if deletion is successful, false otherwise - * @throws ApiException if there is an issue with the API call */ @Override - public boolean brapiDelete(List members) throws ApiException { + public boolean brapiDelete(List members) { // TODO: implement delete list for BrAPIJavaTestServer return false; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java index 0076e686e..0287f8ce1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java @@ -55,10 +55,9 @@ public PendingGermplasm(AppendOverwriteMiddlewareContext context, * * @param members List of entities to be created * @return List of created entities - * @throws ApiException if there is an issue with the API call */ @Override - public List brapiPost(List members) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + public List brapiPost(List members) { return null; } @@ -82,11 +81,10 @@ public List brapiRead() throws ApiException { * * @param members List of entities to be updated * @return List of updated entities - * @throws ApiException if there is an issue with the API call * @throws IllegalArgumentException if method arguments are invalid */ @Override - public List brapiPut(List members) throws ApiException, IllegalArgumentException { + public List brapiPut(List members) throws IllegalArgumentException { return new ArrayList<>(); } @@ -95,10 +93,9 @@ public List brapiPut(List members) throws ApiException, IllegalArgumen * * @param members List of entities to be deleted * @return true if deletion is successful, false otherwise - * @throws ApiException if there is an issue with the API call */ @Override - public boolean brapiDelete(List members) throws ApiException { + public boolean brapiDelete(List members) { return false; } From 58d3a7f9101cddc744c80b65c4d307c790a2c486 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 11 Jul 2024 09:11:29 -0400 Subject: [PATCH 201/203] [BI-2134] - removed uses of deprecated constant all tests passing --- .../factory/entity/PendingDataset.java | 10 ++++++---- .../PopulateExistingPendingImportObjectsStep.java | 11 +++++++---- .../steps/PopulateNewPendingImportObjectsStep.java | 7 +++++-- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java index a5d3e4ca2..216006f23 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java @@ -18,6 +18,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; import io.micronaut.context.annotation.Prototype; +import com.google.gson.JsonArray; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.request.BrAPIListNewRequest; @@ -34,6 +35,7 @@ import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.DatasetUtil; import java.util.ArrayList; import java.util.List; @@ -102,10 +104,10 @@ public List brapiPost(List members) throws A @Override public List brapiRead() throws ApiException { // Get the id of the dataset belonging to the required exp units - String datasetId = cache.getTrialByNameNoScope().values().iterator().next().getBrAPIObject() + JsonArray datasetsJson = cache.getTrialByNameNoScope().values().iterator().next().getBrAPIObject() .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) - .getAsString(); + .getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); + String datasetId = DatasetUtil.getTopLevelDatasetFromJson(datasetsJson).getId().toString(); // Get the dataset belonging to required exp units return List.of(datasetService.fetchDatasetById(datasetId, importContext.getProgram()).orElseThrow(ApiException::new)); @@ -234,7 +236,7 @@ public void initializeWorkflow(List members) { e -> { if (cache.getPendingTrialByOUId().isEmpty() || pendingDatasetByName.isEmpty() || - !cache.getPendingTrialByOUId().values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + cache.getPendingTrialByOUId().values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { throw new IllegalStateException("There is not an observation data set for this unit: " + e.getKey()); } return pendingDatasetByName.values().iterator().next(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index c5196e662..ab4233b0a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -16,6 +16,7 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; +import com.google.gson.JsonArray; import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; @@ -47,6 +48,7 @@ import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.model.Trait; import org.breedinginsight.services.ProgramLocationService; +import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; @@ -373,11 +375,12 @@ private Map> initializeObsVarDatas Optional> trialPIO = getTrialPIO(experimentImportRows, trialByNameNoScope); - if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { - String datasetId = trialPIO.get().getBrAPIObject() + if (trialPIO.isPresent() && !trialPIO.get().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { + JsonArray datasetsJson = trialPIO.get().getBrAPIObject() .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) - .getAsString(); + .getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); + String datasetId = DatasetUtil.getTopLevelDatasetFromJson(datasetsJson).getId().toString(); + try { List existingDatasets = brAPIListDAO .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index a471b84f3..3f523d6a1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -17,6 +17,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; import com.google.gson.Gson; +import com.google.gson.JsonArray; import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpStatus; import io.micronaut.http.exceptions.HttpStatusException; @@ -52,6 +53,7 @@ import org.breedinginsight.model.User; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; import tech.tablesaw.api.Table; @@ -521,9 +523,10 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Import UUID trialID = trialPIO.getId(); UUID datasetId = null; if (commit) { - datasetId = UUID.fromString(trialPIO.getBrAPIObject() + JsonArray datasetsArray = trialPIO.getBrAPIObject() .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + .getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); + datasetId = DatasetUtil.getTopLevelDatasetFromJson(datasetsArray).getId(); } PendingImportObject studyPIO = studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); From 578d1270f2d6631bc552d02d5e0313cf40b0f356 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:14:23 -0400 Subject: [PATCH 202/203] delete unused interface --- .../brapps/importer/model/Workflow.java | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/Workflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/Workflow.java deleted file mode 100644 index 807f8387f..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/Workflow.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.breedinginsight.brapps.importer.model; - -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; - -public interface Workflow { - - ProcessedData process(ImportContext context); - String getName(); -} From 5ecc421a203dab379d3b74da2eb4b10483f39eea Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:18:00 -0400 Subject: [PATCH 203/203] delete unused class --- .../steps/PopulateModifiedPendingImportObjectsStep.java | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java deleted file mode 100644 index 3fe0ba4ee..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow.steps; - -// TODO: think this would be for other workflow only -public class PopulateModifiedPendingImportObjectsStep { -}