From 314d0b129d6f563b2b2bf34cbebe51f8fb7bf0a5 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 6 May 2024 17:07:01 -0400 Subject: [PATCH 01/50] Added db migration to create workflows --- .../V1.22.0__add_experiment_workflows.sql | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql diff --git a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql new file mode 100644 index 000000000..a78947c38 --- /dev/null +++ b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql @@ -0,0 +1,41 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE importer_mapping_workflow +( + like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, + mapping_id UUID NOT NULL, + name TEXT +); + +ALTER TABLE importer_mapping_workflow + ADD FOREIGN KEY (mapping_id) REFERENCES importer_mapping (id); + +DO +$$ +DECLARE + exp_mapping_id UUID; +BEGIN + exp_mapping_id := (SELECT id FROM importer_mapping WHERE name = 'ExperimentsTemplateMap'); + +INSERT INTO public.importer_mapping_workflow (mapping_id, name) +VALUES + (exp_mapping_id, 'Create new experiment'), + (exp_mapping_id, 'Append experimental dataset'), + (exp_mapping_id, 'Create new experimental environment'); +END +$$; \ No newline at end of file From 8fc2f31a33698340a15d42dd8c80da7561c76684 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 7 May 2024 11:33:56 -0400 Subject: [PATCH 02/50] Added bean mapping for factory producing appropriate workflow --- .../migration/V1.22.0__add_experiment_workflows.sql | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql index a78947c38..d5c4b9556 100644 --- a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql +++ b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql @@ -19,7 +19,8 @@ CREATE TABLE importer_mapping_workflow ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, mapping_id UUID NOT NULL, - name TEXT + name TEXT NOT NULL, + bean TEXT NOT NULL ); ALTER TABLE importer_mapping_workflow @@ -32,10 +33,10 @@ DECLARE BEGIN exp_mapping_id := (SELECT id FROM importer_mapping WHERE name = 'ExperimentsTemplateMap'); -INSERT INTO public.importer_mapping_workflow (mapping_id, name) +INSERT INTO public.importer_mapping_workflow (mapping_id, name, bean) VALUES - (exp_mapping_id, 'Create new experiment'), - (exp_mapping_id, 'Append experimental dataset'), - (exp_mapping_id, 'Create new experimental environment'); + (exp_mapping_id, 'Create new experiment', 'CreateNewExperimentWorkflow'), + (exp_mapping_id, 'Append experimental dataset', 'AppendOverwritePhenotypesWorkflow'), + (exp_mapping_id, 'Create new experimental environment', 'CreateNewEnvironmentWorkflow'); END $$; \ No newline at end of file From 4b61708eaa9ece75443f6608b20bdf389b165c93 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 7 May 2024 11:36:12 -0400 Subject: [PATCH 03/50] Added endpoint for retrieving workflows for given mapping id --- .../controllers/ImportController.java | 18 +++++++ .../daos/ImportMappingWorkflowDAO.java | 46 ++++++++++++++++ .../model/workflow/ImportMappingWorkflow.java | 52 +++++++++++++++++++ .../importer/services/FileImportService.java | 14 ++++- 4 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index e0a61117a..5c9ac0ca1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -37,6 +37,7 @@ import org.breedinginsight.api.model.v1.response.metadata.StatusCode; import org.breedinginsight.api.v1.controller.metadata.AddMetadata; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; +import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; import org.breedinginsight.brapps.importer.services.ImportConfigManager; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; import org.breedinginsight.brapps.importer.services.FileImportService; @@ -208,4 +209,21 @@ public HttpResponse>> getSystemMappings(@Nu Response> response = new Response(metadata, new DataResponse<>(result)); return HttpResponse.ok(response); } + + @Get("/import/mappings/{mappingId}/workflows") + @Produces(MediaType.APPLICATION_JSON) + @AddMetadata + @Secured(SecurityRule.IS_ANONYMOUS) + public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { + + List workflows = fileImportService.getWorkflowsForSystemMapping(mappingId); + + List metadataStatus = new ArrayList<>(); + metadataStatus.add(new Status(StatusCode.INFO, "Successful Query")); + Pagination pagination = new Pagination(workflows.size(), workflows.size(), 1, 0); + Metadata metadata = new Metadata(pagination, metadataStatus); + + Response> response = new Response(metadata, new DataResponse<>(workflows)); + return HttpResponse.ok(response); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java new file mode 100644 index 000000000..e6a0a7104 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java @@ -0,0 +1,46 @@ +package org.breedinginsight.brapps.importer.daos; + +import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; +import org.breedinginsight.dao.db.tables.daos.ImporterMappingWorkflowDao; +import org.jooq.Configuration; +import org.jooq.DSLContext; +import org.jooq.Record; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static org.breedinginsight.dao.db.Tables.*; + +public class ImportMappingWorkflowDAO extends ImporterMappingWorkflowDao { + + private DSLContext dsl; + + @Inject + public ImportMappingWorkflowDAO(Configuration config, DSLContext dsl) { + super(config); + this.dsl = dsl; + } + + /** + * Retrieves a list of ImportMappingWorkflow objects associated with the given mappingId. + * + * @param mappingId The UUID of the mapping to retrieve the workflows for. + * @return A list of ImportMappingWorkflow objects. + */ + public List getWorkflowsByImportMappingId(UUID mappingId) { + + List records = dsl.select() + .from(IMPORTER_MAPPING_WORKFLOW) + .where(IMPORTER_MAPPING_WORKFLOW.MAPPING_ID.eq(mappingId)) + .fetch(); + + List workflows = new ArrayList<>(); + for (Record record: records) { + workflows.add(ImportMappingWorkflow.parseSQLRecord(record)); + } + return workflows; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java new file mode 100644 index 000000000..51a70bb5a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java @@ -0,0 +1,52 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; +import lombok.experimental.SuperBuilder; + +import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; +import org.jooq.Record; + +import static org.breedinginsight.dao.db.tables.ImporterMappingWorkflowTable.IMPORTER_MAPPING_WORKFLOW; + +@Getter +@Setter +@Accessors(chain=true) +@ToString +@NoArgsConstructor +@SuperBuilder() +public class ImportMappingWorkflow extends ImporterMappingWorkflowEntity { + + public ImportMappingWorkflow(ImporterMappingWorkflowEntity importMappingWorkflowEntity) { + this.setId(importMappingWorkflowEntity.getId()); + this.setName(importMappingWorkflowEntity.getName()); + } + + public static ImportMappingWorkflow parseSQLRecord(Record record) { + + return ImportMappingWorkflow.builder() + .id(record.getValue(IMPORTER_MAPPING_WORKFLOW.ID)) + .name(record.getValue(IMPORTER_MAPPING_WORKFLOW.NAME)) + .build(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 05c48601d..950465459 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.micronaut.http.HttpStatus; +import io.micronaut.http.annotation.PathVariable; import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.multipart.CompletedFileUpload; import io.micronaut.http.server.exceptions.InternalServerException; @@ -32,6 +33,7 @@ import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.daos.ImportMappingProgramDAO; +import org.breedinginsight.brapps.importer.daos.ImportMappingWorkflowDAO; import org.breedinginsight.brapps.importer.model.ImportProgress; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; @@ -40,8 +42,10 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; +import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingProgramEntity; +import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import org.breedinginsight.services.ProgramService; @@ -80,12 +84,13 @@ public class FileImportService { private final ImportDAO importDAO; private final DSLContext dsl; private final ImportMappingProgramDAO importMappingProgramDAO; + private final ImportMappingWorkflowDAO importMappingWorkflowDAO; @Inject FileImportService(ProgramUserService programUserService, ProgramService programService, MimeTypeParser mimeTypeParser, ImportMappingDAO importMappingDAO, ObjectMapper objectMapper, MappingManager mappingManager, ImportConfigManager configManager, ImportDAO importDAO, DSLContext dsl, ImportMappingProgramDAO importMappingProgramDAO, - UserService userService) { + ImportMappingWorkflowDAO importMappingWorkflowDAO, UserService userService) { this.programUserService = programUserService; this.programService = programService; this.mimeTypeParser = mimeTypeParser; @@ -97,6 +102,7 @@ public class FileImportService { this.dsl = dsl; this.importMappingProgramDAO = importMappingProgramDAO; this.userService = userService; + this.importMappingWorkflowDAO = importMappingWorkflowDAO; } public List getAllImportTypeConfigs() { @@ -559,4 +565,10 @@ public List getSystemMappingByName(String name) { List importMappings = importMappingDAO.getSystemMappingByName(name); return importMappings; } + + public List getWorkflowsForSystemMapping(UUID mappingId) { + return importMappingWorkflowDAO.getWorkflowsByImportMappingId(mappingId); + } + + } From c1fe8fa55d4b4170cf52c3b82fef2f18f5c5c765 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 7 May 2024 17:25:02 -0400 Subject: [PATCH 04/50] Simplify record mapping --- .../importer/daos/ImportMappingWorkflowDAO.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java index e6a0a7104..c2ad58c23 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java @@ -30,17 +30,10 @@ public ImportMappingWorkflowDAO(Configuration config, DSLContext dsl) { * @return A list of ImportMappingWorkflow objects. */ public List getWorkflowsByImportMappingId(UUID mappingId) { - - List records = dsl.select() + return dsl.select() .from(IMPORTER_MAPPING_WORKFLOW) .where(IMPORTER_MAPPING_WORKFLOW.MAPPING_ID.eq(mappingId)) - .fetch(); - - List workflows = new ArrayList<>(); - for (Record record: records) { - workflows.add(ImportMappingWorkflow.parseSQLRecord(record)); - } - return workflows; + .fetch(ImportMappingWorkflow::parseSQLRecord); } } From 3ca1a30a4ddc76a172d14d2685e242ed4ada7d2b Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 10 May 2024 12:05:09 -0400 Subject: [PATCH 05/50] Pass workflow id through to import services --- .../controllers/UploadController.java | 57 ++++++++++++++++++- .../model/imports/BrAPIImportService.java | 13 +---- .../model/imports/ImportServiceContext.java | 25 ++++++++ .../ExperimentImportService.java | 15 ++++- .../germplasm/GermplasmImportService.java | 11 +++- .../sample/SampleSubmissionImportService.java | 16 +++--- .../importer/services/FileImportService.java | 22 +++++-- 7 files changed, 128 insertions(+), 31 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java index f9d55bd20..053f1dc3c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java @@ -114,7 +114,7 @@ public HttpResponse> commitData(@PathVariable UUID prog @PathVariable UUID uploadId, @Body @Nullable Map userInput) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, actingUser, userInput, true); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, null, actingUser, userInput, true); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { @@ -140,7 +140,60 @@ public HttpResponse> previewData(@PathVariable UUID pro @PathVariable UUID uploadId) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, actingUser, null, false); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, null, actingUser, null, false); + Response response = new Response(result); + return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + return HttpResponse.notFound(); + } catch (AuthorizationException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.FORBIDDEN, e.getMessage()); + } catch (UnprocessableEntityException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } catch (HttpStatusException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(e.getStatus(), e.getMessage()); + } + } + + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/preview") + @Produces(MediaType.APPLICATION_JSON) + @AddMetadata + @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) + public HttpResponse> previewData(@PathVariable UUID programId, @PathVariable UUID mappingId, + @PathVariable UUID workflowId, @PathVariable UUID uploadId) { + try { + AuthenticatedUser actingUser = securityService.getUser(); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, null, false); + Response response = new Response(result); + return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + return HttpResponse.notFound(); + } catch (AuthorizationException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.FORBIDDEN, e.getMessage()); + } catch (UnprocessableEntityException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } catch (HttpStatusException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(e.getStatus(), e.getMessage()); + } + } + + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/commit") + @Produces(MediaType.APPLICATION_JSON) + @AddMetadata + @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) + public HttpResponse> commitData(@PathVariable UUID programId, @PathVariable UUID mappingId, + @PathVariable UUID workflowId, @PathVariable UUID uploadId, + @Body @Nullable Map userInput) { + try { + AuthenticatedUser actingUser = securityService.getUser(); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, userInput, true); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index 1d520371c..d06454c30 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -17,18 +17,7 @@ package org.breedinginsight.brapps.importer.model.imports; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.ValidatorException; -import tech.tablesaw.api.Table; - -import java.util.List; public interface BrAPIImportService { String getImportTypeId(); @@ -48,6 +37,6 @@ default String getMissingUserInputMsg(String fieldName) { default String getWrongUserInputDataTypeMsg(String fieldName, String typeName) { return String.format("User input, \"%s\" must be an %s", fieldName, typeName); } - ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + ImportPreviewResponse process(ImportServiceContext context) throws Exception; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java new file mode 100644 index 000000000..ef7639c2f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -0,0 +1,25 @@ +package org.breedinginsight.brapps.importer.model.imports; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import tech.tablesaw.api.Table; +import java.util.List; +import java.util.UUID; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ImportServiceContext { + private UUID workflowId; + private List brAPIImports; + private Table data; + private Program program; + private ImportUpload upload; + private User user; + private boolean commit; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index cd795564a..965a7d20f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -21,6 +21,7 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; @@ -66,12 +67,22 @@ public String getMissingColumnMsg(String columnName) { } @Override - public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + public ImportPreviewResponse process(ImportServiceContext context) throws Exception { ImportPreviewResponse response = null; List processors = List.of(experimentProcessorProvider.get()); - response = processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); + // TODO: change to calling process directly on processor (not using processor manager and pass along workflowId) + if (context.getWorkflowId() != null) { + log.info("Workflow UUID: " + context.getWorkflowId()); + } + response = processorManagerProvider.get().process(context.getBrAPIImports(), + processors, + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); return response; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index b4eac6b96..ce52f483f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -21,6 +21,7 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.services.processors.GermplasmProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; @@ -62,12 +63,18 @@ public String getImportTypeId() { } @Override - public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + public ImportPreviewResponse process(ImportServiceContext context) throws Exception { ImportPreviewResponse response = null; List processors = List.of(germplasmProcessorProvider.get()); - response = processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); + response = processorManagerProvider.get().process(context.getBrAPIImports(), + processors, + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); return response; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java index 434626e68..2c8e8b7b5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java @@ -21,6 +21,7 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; @@ -59,13 +60,14 @@ public BrAPIImport getImportClass() { } @Override - public ImportPreviewResponse process(List brAPIImports, - Table data, - Program program, - ImportUpload upload, - User user, - Boolean commit) throws Exception { + public ImportPreviewResponse process(ImportServiceContext context) throws Exception { List processors = List.of(sampleProcessorProvider.get()); - return processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); + return processorManagerProvider.get().process(context.getBrAPIImports(), + processors, + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 950465459..960a8077f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -38,6 +38,7 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.response.ImportResponse; @@ -328,7 +329,7 @@ public ImportResponse uploadData(UUID programId, UUID mappingId, AuthenticatedUs return response; } - public ImportResponse updateUpload(UUID programId, UUID uploadId, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws + public ImportResponse updateUpload(UUID programId, UUID uploadId, UUID workflowId, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws DoesNotExistException, UnprocessableEntityException, AuthorizationException { Program program = validateRequest(programId, actingUser); @@ -378,7 +379,7 @@ public ImportResponse updateUpload(UUID programId, UUID uploadId, AuthenticatedU } else { brAPIImportList = mappingManager.map(mappingConfig, data); } - processFile(brAPIImportList, data, program, upload, user, commit, importService, actingUser); + processFile(workflowId, brAPIImportList, data, program, upload, user, commit, importService, actingUser); } catch (UnprocessableEntityException e) { log.error(e.getMessage(), e); ImportProgress progress = upload.getProgress(); @@ -424,13 +425,22 @@ public ImportUpload setDynamicColumns(ImportUpload newUpload, Table data, Import return newUpload; } - private void processFile(List finalBrAPIImportList, Table data, Program program, - ImportUpload upload, User user, Boolean commit, BrAPIImportService importService, - AuthenticatedUser actingUser) { + private void processFile(UUID workflowId, List finalBrAPIImportList, Table data, Program program, + ImportUpload upload, User user, Boolean commit, BrAPIImportService importService, + AuthenticatedUser actingUser) { // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { - importService.process(finalBrAPIImportList, data, program, upload, user, commit); + ImportServiceContext context = ImportServiceContext.builder() + .workflowId(workflowId) + .brAPIImports(finalBrAPIImportList) + .data(data) + .program(program) + .upload(upload) + .user(user) + .commit(commit) + .build(); + importService.process(context); } catch (UnprocessableEntityException e) { log.error(e.getMessage(), e); ImportProgress progress = upload.getProgress(); From c8e0bfe736957eb5b3ddd5186cd7ab6096580c4a Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 13 May 2024 16:45:25 -0400 Subject: [PATCH 06/50] Add workflow factory to create workflows from ids --- .../daos/ImportMappingWorkflowDAO.java | 12 +++ .../model/imports/ImportServiceContext.java | 21 ++++- .../ExperimentImportService.java | 8 +- .../model/workflow/ImportContext.java | 47 +++++++++++ .../model/workflow/ImportMappingWorkflow.java | 2 - .../model/workflow/ProcessedData.java | 31 +++++++ .../importer/model/workflow/Workflow.java | 36 ++++++++ .../importer/services/FileImportService.java | 14 +++- .../workflow/CreateNewExperimentWorkflow.java | 50 +++++++++++ .../services/workflow/WorkflowFactory.java | 84 +++++++++++++++++++ 10 files changed, 296 insertions(+), 9 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java index c2ad58c23..21aca395a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java @@ -9,6 +9,7 @@ import javax.inject.Inject; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.UUID; import static org.breedinginsight.dao.db.Tables.*; @@ -36,4 +37,15 @@ public List getWorkflowsByImportMappingId(UUID mappingId) .fetch(ImportMappingWorkflow::parseSQLRecord); } + /** + * Retrieves a workflow by its ID. + * + * @param workflowId The ID of the workflow to retrieve. + * @return An Optional containing the ImportMappingWorkflow if found, otherwise an empty Optional. + */ + public Optional getWorkflowById(UUID workflowId) { + return Optional.ofNullable(fetchOneById(workflowId)) + .map(ImportMappingWorkflow::new); + } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java index ef7639c2f..45393f67c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -1,12 +1,29 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.model.imports; import lombok.*; import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import tech.tablesaw.api.Table; import java.util.List; -import java.util.UUID; @Getter @Setter @@ -15,7 +32,7 @@ @AllArgsConstructor @NoArgsConstructor public class ImportServiceContext { - private UUID workflowId; + private Workflow workflow; private List brAPIImports; private Table data; private Program program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 965a7d20f..c6f68b251 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -72,10 +72,12 @@ public ImportPreviewResponse process(ImportServiceContext context) ImportPreviewResponse response = null; List processors = List.of(experimentProcessorProvider.get()); - // TODO: change to calling process directly on processor (not using processor manager and pass along workflowId) - if (context.getWorkflowId() != null) { - log.info("Workflow UUID: " + context.getWorkflowId()); + + if (context.getWorkflow() != null) { + log.info("Workflow: " + context.getWorkflow().getName()); } + + // TODO: change to calling workflow process instead of processor manager response = processorManagerProvider.get().process(context.getBrAPIImports(), processors, context.getData(), diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java new file mode 100644 index 000000000..121ef6b19 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java @@ -0,0 +1,47 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import tech.tablesaw.api.Table; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ImportContext { + private UUID workflowId; + private ImportUpload upload; + private List importRows; + private Map mappedBrAPIImport; + private Table data; + private Program program; + private User user; + private boolean commit; +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java index 51a70bb5a..0a23e1aae 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java @@ -36,12 +36,10 @@ @NoArgsConstructor @SuperBuilder() public class ImportMappingWorkflow extends ImporterMappingWorkflowEntity { - public ImportMappingWorkflow(ImporterMappingWorkflowEntity importMappingWorkflowEntity) { this.setId(importMappingWorkflowEntity.getId()); this.setName(importMappingWorkflowEntity.getName()); } - public static ImportMappingWorkflow parseSQLRecord(Record record) { return ImportMappingWorkflow.builder() diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java new file mode 100644 index 000000000..e73d6ad48 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -0,0 +1,31 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; + +import java.util.Map; + +@Data +@Builder +@ToString +@NoArgsConstructor +public class ProcessedData { + private Map statistics; +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java new file mode 100644 index 000000000..9ea0e7bf9 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -0,0 +1,36 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +public interface Workflow { + + /** + * Processes the given import context and returns the processed data. + * + * @param context the import context containing the necessary data for processing + * @return the processed data + */ + ProcessedData process(ImportContext context); + + /** + * Retrieves the name of the Workflow for logging display purposes. + * + * @return the name of the Workflow + */ + String getName(); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 960a8077f..2c3f7854f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -44,6 +44,8 @@ import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.workflow.WorkflowFactory; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingProgramEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; @@ -86,12 +88,13 @@ public class FileImportService { private final DSLContext dsl; private final ImportMappingProgramDAO importMappingProgramDAO; private final ImportMappingWorkflowDAO importMappingWorkflowDAO; + private final WorkflowFactory workflowFactory; @Inject FileImportService(ProgramUserService programUserService, ProgramService programService, MimeTypeParser mimeTypeParser, ImportMappingDAO importMappingDAO, ObjectMapper objectMapper, MappingManager mappingManager, ImportConfigManager configManager, ImportDAO importDAO, DSLContext dsl, ImportMappingProgramDAO importMappingProgramDAO, - ImportMappingWorkflowDAO importMappingWorkflowDAO, UserService userService) { + ImportMappingWorkflowDAO importMappingWorkflowDAO, WorkflowFactory workflowFactory, UserService userService) { this.programUserService = programUserService; this.programService = programService; this.mimeTypeParser = mimeTypeParser; @@ -104,6 +107,7 @@ public class FileImportService { this.importMappingProgramDAO = importMappingProgramDAO; this.userService = userService; this.importMappingWorkflowDAO = importMappingWorkflowDAO; + this.workflowFactory = workflowFactory; } public List getAllImportTypeConfigs() { @@ -431,8 +435,14 @@ private void processFile(UUID workflowId, List finalBrAPIImportList // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { + Workflow workflow = null; + if (workflowId != null) { + Optional optionalWorkflow = workflowFactory.getWorkflow(workflowId); + workflow = optionalWorkflow.orElse(null); + } + ImportServiceContext context = ImportServiceContext.builder() - .workflowId(workflowId) + .workflow(workflow) .brAPIImports(finalBrAPIImportList) .data(data) .program(program) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java new file mode 100644 index 000000000..6f6ab661a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -0,0 +1,50 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; + +import javax.inject.Named; + +/** + * This class represents a workflow for creating a new experiment. The bean name must match the appropriate bean column + * value in the import_mapping_workflow db table + */ +@Prototype +@Named("CreateNewExperimentWorkflow") +public class CreateNewExperimentWorkflow implements Workflow { + + @Override + public ProcessedData process(ImportContext context) { + // TODO + return null; + } + + /** + * Retrieves the name of the workflow. This is used for logging display purposes. + * + * @return the name of the workflow + */ + @Override + public String getName() { + return "CreateNewExperimentWorkflow"; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java new file mode 100644 index 000000000..60a2fd884 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java @@ -0,0 +1,84 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.workflow; + +import io.micronaut.context.ApplicationContext; +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.exceptions.NoSuchBeanException; +import io.micronaut.inject.qualifiers.Qualifiers; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.daos.ImportMappingWorkflowDAO; +import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; + +import javax.inject.Inject; +import java.util.Optional; +import java.util.UUID; + +@Factory +@Slf4j +public class WorkflowFactory { + + private final ImportMappingWorkflowDAO importMappingWorkflowDAO; + private final ApplicationContext applicationContext; + + @Inject + public WorkflowFactory(ImportMappingWorkflowDAO importMappingWorkflowDAO, + ApplicationContext applicationContext) { + this.importMappingWorkflowDAO = importMappingWorkflowDAO; + this.applicationContext = applicationContext; + } + + /** + * Produces the appropriate workflow instance based on the import context + * + * @param context the import context + * @return an Optional containing the workflow if found, otherwise an empty Optional + * + * @throws IllegalStateException + * @throws NoSuchBeanException + */ + public Optional getWorkflow(UUID workflowId) { + + if (workflowId != null) { + // construct workflow from db record + Optional workflowOptional = importMappingWorkflowDAO.getWorkflowById(workflowId); + if (workflowOptional.isPresent()) { + ImportMappingWorkflow importMappingworkflow = workflowOptional.orElseThrow(() -> { + String msg = "Must have record in db for workflowId"; + log.error(msg); + return new IllegalStateException(msg); + }); + + // newer versions of micronaut have fancier ways to do this using annotations with provider but as + // far as I can tell it's not available in 2.5 + Workflow workflow; + try { + workflow = applicationContext.getBean(Workflow.class, Qualifiers.byName(importMappingworkflow.getBean())); + } catch (NoSuchBeanException e) { + log.error("Could not find workflow class implementation for bean: " + importMappingworkflow.getBean()); + throw e; + } + + return Optional.of(workflow); + } + } + + return Optional.empty(); + } +} From 4391479ee6e2d257e0713c6c6b7cbfc631ee030c Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 13 May 2024 16:50:45 -0400 Subject: [PATCH 07/50] Add missing file header --- .../daos/ImportMappingWorkflowDAO.java | 19 +++++++++++++++++-- .../importer/services/FileImportService.java | 2 +- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java index 21aca395a..216506b5c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java @@ -1,13 +1,28 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.daos; import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; import org.breedinginsight.dao.db.tables.daos.ImporterMappingWorkflowDao; import org.jooq.Configuration; import org.jooq.DSLContext; -import org.jooq.Record; import javax.inject.Inject; -import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.UUID; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 2c3f7854f..ef6c5f884 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -440,7 +440,7 @@ private void processFile(UUID workflowId, List finalBrAPIImportList Optional optionalWorkflow = workflowFactory.getWorkflow(workflowId); workflow = optionalWorkflow.orElse(null); } - + ImportServiceContext context = ImportServiceContext.builder() .workflow(workflow) .brAPIImports(finalBrAPIImportList) From 87ffdb53d134b43b0955aacee063c351d29d5cfd Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 13 May 2024 17:08:39 -0400 Subject: [PATCH 08/50] Add bean to import mapping workflow model --- .../brapps/importer/model/workflow/ImportMappingWorkflow.java | 4 ++++ .../brapps/importer/model/workflow/ProcessedData.java | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java index 0a23e1aae..4ea2a888b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java @@ -36,15 +36,19 @@ @NoArgsConstructor @SuperBuilder() public class ImportMappingWorkflow extends ImporterMappingWorkflowEntity { + + public ImportMappingWorkflow(ImporterMappingWorkflowEntity importMappingWorkflowEntity) { this.setId(importMappingWorkflowEntity.getId()); this.setName(importMappingWorkflowEntity.getName()); + this.setBean(importMappingWorkflowEntity.getBean()); } public static ImportMappingWorkflow parseSQLRecord(Record record) { return ImportMappingWorkflow.builder() .id(record.getValue(IMPORTER_MAPPING_WORKFLOW.ID)) .name(record.getValue(IMPORTER_MAPPING_WORKFLOW.NAME)) + .bean(record.getValue(IMPORTER_MAPPING_WORKFLOW.BEAN)) .build(); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java index e73d6ad48..f9f8196c2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -23,7 +23,6 @@ import java.util.Map; @Data -@Builder @ToString @NoArgsConstructor public class ProcessedData { From 1c1df39b03dd513adff46bc8bd7e8d9dde19bff7 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 14 May 2024 10:29:04 -0400 Subject: [PATCH 09/50] Added position column to workflows for explicit ordering --- .../daos/ImportMappingWorkflowDAO.java | 4 +++- .../model/workflow/ImportMappingWorkflow.java | 2 ++ .../services/workflow/WorkflowFactory.java | 4 +++- .../V1.22.0__add_experiment_workflows.sql | 19 ++++++++++++++----- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java index 216506b5c..1b60bda45 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java @@ -40,7 +40,8 @@ public ImportMappingWorkflowDAO(Configuration config, DSLContext dsl) { } /** - * Retrieves a list of ImportMappingWorkflow objects associated with the given mappingId. + * Retrieves a list of ImportMappingWorkflow objects associated with the given mappingId. They are ordered by + * position for proper ordering on the front end. * * @param mappingId The UUID of the mapping to retrieve the workflows for. * @return A list of ImportMappingWorkflow objects. @@ -49,6 +50,7 @@ public List getWorkflowsByImportMappingId(UUID mappingId) return dsl.select() .from(IMPORTER_MAPPING_WORKFLOW) .where(IMPORTER_MAPPING_WORKFLOW.MAPPING_ID.eq(mappingId)) + .orderBy(IMPORTER_MAPPING_WORKFLOW.POSITION.asc()) .fetch(ImportMappingWorkflow::parseSQLRecord); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java index 4ea2a888b..8dc5800bc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java @@ -42,6 +42,7 @@ public ImportMappingWorkflow(ImporterMappingWorkflowEntity importMappingWorkflow this.setId(importMappingWorkflowEntity.getId()); this.setName(importMappingWorkflowEntity.getName()); this.setBean(importMappingWorkflowEntity.getBean()); + this.setPosition(importMappingWorkflowEntity.getPosition()); } public static ImportMappingWorkflow parseSQLRecord(Record record) { @@ -49,6 +50,7 @@ public static ImportMappingWorkflow parseSQLRecord(Record record) { .id(record.getValue(IMPORTER_MAPPING_WORKFLOW.ID)) .name(record.getValue(IMPORTER_MAPPING_WORKFLOW.NAME)) .bean(record.getValue(IMPORTER_MAPPING_WORKFLOW.BEAN)) + .position(record.getValue(IMPORTER_MAPPING_WORKFLOW.POSITION)) .build(); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java index 60a2fd884..d265b66a0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java @@ -48,7 +48,7 @@ public WorkflowFactory(ImportMappingWorkflowDAO importMappingWorkflowDAO, * Produces the appropriate workflow instance based on the import context * * @param context the import context - * @return an Optional containing the workflow if found, otherwise an empty Optional + * @return an Optional containing the workflow if id is not null, otherwise an empty Optional * * @throws IllegalStateException * @throws NoSuchBeanException @@ -58,6 +58,8 @@ public Optional getWorkflow(UUID workflowId) { if (workflowId != null) { // construct workflow from db record Optional workflowOptional = importMappingWorkflowDAO.getWorkflowById(workflowId); + + // TODO: look at this optional handling if (workflowOptional.isPresent()) { ImportMappingWorkflow importMappingworkflow = workflowOptional.orElseThrow(() -> { String msg = "Must have record in db for workflowId"; diff --git a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql index d5c4b9556..2c9d4d547 100644 --- a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql +++ b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql @@ -15,12 +15,21 @@ * limitations under the License. */ +/** + Table maps workflows to import mappings and provides required configuration options + + mapping_id - link to importer_mapping that this provides a workflow for + name - name that will be displayed on front end + bean - must match @Named("") annotation on Workflow class + position - for ordering records explicitly, wanted for front end default option and order + */ CREATE TABLE importer_mapping_workflow ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, mapping_id UUID NOT NULL, name TEXT NOT NULL, - bean TEXT NOT NULL + bean TEXT NOT NULL, + position INTEGER NOT NULL ); ALTER TABLE importer_mapping_workflow @@ -33,10 +42,10 @@ DECLARE BEGIN exp_mapping_id := (SELECT id FROM importer_mapping WHERE name = 'ExperimentsTemplateMap'); -INSERT INTO public.importer_mapping_workflow (mapping_id, name, bean) +INSERT INTO public.importer_mapping_workflow (mapping_id, name, bean, position) VALUES - (exp_mapping_id, 'Create new experiment', 'CreateNewExperimentWorkflow'), - (exp_mapping_id, 'Append experimental dataset', 'AppendOverwritePhenotypesWorkflow'), - (exp_mapping_id, 'Create new experimental environment', 'CreateNewEnvironmentWorkflow'); + (exp_mapping_id, 'Create new experiment', 'CreateNewExperimentWorkflow', 0), + (exp_mapping_id, 'Append experimental dataset', 'AppendOverwritePhenotypesWorkflow', 1), + (exp_mapping_id, 'Create new experimental environment', 'CreateNewEnvironmentWorkflow', 2); END $$; \ No newline at end of file From 36944a510a2dade39bd1dff565621a1b2f6f6e41 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 14 May 2024 10:45:35 -0400 Subject: [PATCH 10/50] Added additional workflow skeletons --- .../AppendOverwritePhenotypesWorkflow.java | 50 +++++++++++++++++++ .../CreateNewEnvironmentWorkflow.java | 50 +++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java new file mode 100644 index 000000000..21df19be6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -0,0 +1,50 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; + +import javax.inject.Named; + +/** + * This class represents a workflow for appending and overwriting phenotypes. The bean name must match the appropriate + * bean column value in the import_mapping_workflow db table + */ + +@Prototype +@Named("AppendOverwritePhenotypesWorkflow") +public class AppendOverwritePhenotypesWorkflow implements Workflow { + @Override + public ProcessedData process(ImportContext context) { + // TODO + return null; + } + + /** + * Retrieves the name of the workflow. This is used for logging display purposes. + * + * @return the name of the workflow + */ + @Override + public String getName() { + return "AppendOverwritePhenotypesWorkflow"; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java new file mode 100644 index 000000000..5776ab59b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java @@ -0,0 +1,50 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; + +import javax.inject.Named; + +/** + * This class represents a workflow for adding new environments to an existing experiment. The bean name must match + * the appropriate bean column value in the import_mapping_workflow db table + */ + +@Prototype +@Named("CreateNewEnvironmentWorkflow") +public class CreateNewEnvironmentWorkflow implements Workflow { + @Override + public ProcessedData process(ImportContext context) { + // TODO + return null; + } + + /** + * Retrieves the name of the workflow. This is used for logging display purposes. + * + * @return the name of the workflow + */ + @Override + public String getName() { + return "CreateNewEnvironmentWorkflow"; + } +} From 7a9af10ed0356559c6a27de474c86697f0e38f58 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 14 May 2024 14:01:41 -0400 Subject: [PATCH 11/50] Clean up factory code --- .../services/workflow/WorkflowFactory.java | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java index d265b66a0..17d6fe64e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java @@ -59,26 +59,23 @@ public Optional getWorkflow(UUID workflowId) { // construct workflow from db record Optional workflowOptional = importMappingWorkflowDAO.getWorkflowById(workflowId); - // TODO: look at this optional handling - if (workflowOptional.isPresent()) { - ImportMappingWorkflow importMappingworkflow = workflowOptional.orElseThrow(() -> { - String msg = "Must have record in db for workflowId"; - log.error(msg); - return new IllegalStateException(msg); - }); + ImportMappingWorkflow importMappingworkflow = workflowOptional.orElseThrow(() -> { + String msg = "Must have record in db for workflowId"; + log.error(msg); + return new IllegalStateException(msg); + }); - // newer versions of micronaut have fancier ways to do this using annotations with provider but as - // far as I can tell it's not available in 2.5 - Workflow workflow; - try { - workflow = applicationContext.getBean(Workflow.class, Qualifiers.byName(importMappingworkflow.getBean())); - } catch (NoSuchBeanException e) { - log.error("Could not find workflow class implementation for bean: " + importMappingworkflow.getBean()); - throw e; - } - - return Optional.of(workflow); + // newer versions of micronaut have fancier ways to do this using annotations with provider but as + // far as I can tell it's not available in 2.5 + Workflow workflow; + try { + workflow = applicationContext.getBean(Workflow.class, Qualifiers.byName(importMappingworkflow.getBean())); + } catch (NoSuchBeanException e) { + log.error("Could not find workflow class implementation for bean: " + importMappingworkflow.getBean()); + throw e; } + + return Optional.of(workflow); } return Optional.empty(); From 0585568cda37b165ae4fddf5dd79635c1d75527e Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 20 May 2024 11:21:01 -0400 Subject: [PATCH 12/50] Moved create workflow work to own branch and rearranged things a little --- .../importer/services/pipeline/Pipeline.java | 35 ++ .../services/pipeline/ProcessingStep.java | 23 + .../experiment/ExperimentUtilities.java | 18 + .../experiment/create/model/PendingData.java | 27 + .../workflow/CreateNewExperimentWorkflow.java | 25 +- .../steps/GetExistingProcessingStep.java | 470 ++++++++++++++++++ .../create/workflow/steps/ProcessStep.java | 15 + .../services/SharedStudyService.java | 120 +++++ .../services/SharedTrialService.java | 180 +++++++ 9 files changed, 912 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/pipeline/Pipeline.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/Pipeline.java b/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/Pipeline.java new file mode 100644 index 000000000..76c4433ee --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/Pipeline.java @@ -0,0 +1,35 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.pipeline; + +public class Pipeline { + + private final ProcessingStep currentStep; + + public Pipeline(ProcessingStep currentStep) { + this.currentStep = currentStep; + } + + public Pipeline addProcessingStep(ProcessingStep newStep) { + return new Pipeline<>(input -> newStep.process(currentStep.process(input))); + } + + public O execute(I input) { + return currentStep.process(input); + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java new file mode 100644 index 000000000..ca16792db --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java @@ -0,0 +1,23 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.breedinginsight.brapps.importer.services.pipeline; + +public interface ProcessingStep { + O process(I input); +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java new file mode 100644 index 000000000..ed685449d --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -0,0 +1,18 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment; + +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; + +import java.util.*; +import java.util.stream.Collectors; + +public class ExperimentUtilities { + + public static final CharSequence COMMA_DELIMITER = ","; + + public static List importRowsToExperimentObservations(List importRows) { + return importRows.stream() + .map(trialImport -> (ExperimentObservation) trialImport) + .collect(Collectors.toList()); + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java new file mode 100644 index 000000000..7d6fc74b9 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -0,0 +1,27 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; + +import lombok.*; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.ProgramLocation; + +import java.util.Map; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class PendingData { + private Map> observationUnitByNameNoScope; + private Map> trialByNameNoScope; + private Map> studyByNameNoScope; + private Map> locationByName; + private Map> obsVarDatasetByName; + private Map> existingGermplasmByGID; +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 6f6ab661a..ebfa5469d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -21,21 +21,42 @@ import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.pipeline.Pipeline; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.GetExistingProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.ProcessStep; +import javax.inject.Inject; import javax.inject.Named; +import javax.inject.Provider; /** * This class represents a workflow for creating a new experiment. The bean name must match the appropriate bean column * value in the import_mapping_workflow db table */ + @Prototype @Named("CreateNewExperimentWorkflow") public class CreateNewExperimentWorkflow implements Workflow { + private final Provider getExistingStepProvider; + private final Provider processStepProvider; + + @Inject + public CreateNewExperimentWorkflow(Provider getExistingStepProvider, + Provider processStepProvider) { + this.getExistingStepProvider = getExistingStepProvider; + this.processStepProvider = processStepProvider; + } + @Override public ProcessedData process(ImportContext context) { // TODO - return null; + Pipeline pipeline = new Pipeline<>(getExistingStepProvider.get()) + .addProcessingStep(processStepProvider.get()); + ProcessedData processed = pipeline.execute(context); + + // TODO: return actual data + return processed; } /** @@ -43,8 +64,10 @@ public ProcessedData process(ImportContext context) { * * @return the name of the workflow */ + @Override public String getName() { return "CreateNewExperimentWorkflow"; } } + diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java new file mode 100644 index 000000000..10afb13a8 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java @@ -0,0 +1,470 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; + +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.BrAPIListTypes; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.*; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedStudyService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedTrialService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.ProgramLocationService; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +/** + * References code common between workflows in shared services. DAO access is done directly in the + * steps rather than another layer of services. + */ + +@Prototype +@Slf4j +public class GetExistingProcessingStep implements ProcessingStep { + + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private final BrAPITrialDAO brAPITrialDAO; + private final BrAPIStudyDAO brAPIStudyDAO; + private final ProgramLocationService locationService; + private final BrAPIListDAO brAPIListDAO; + private final BrAPIGermplasmDAO brAPIGermplasmDAO; + private final SharedStudyService sharedStudyService; + private final SharedTrialService sharedTrialService; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public GetExistingProcessingStep(BrAPIObservationUnitDAO brAPIObservationUnitDAO, + BrAPITrialDAO brAPITrialDAO, + BrAPIStudyDAO brAPIStudyDAO, + ProgramLocationService locationService, + BrAPIListDAO brAPIListDAO, + BrAPIGermplasmDAO brAPIGermplasmDAO, + SharedStudyService sharedStudyService, + SharedTrialService sharedTrialService) { + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + this.brAPITrialDAO = brAPITrialDAO; + this.brAPIStudyDAO = brAPIStudyDAO; + this.locationService = locationService; + this.brAPIListDAO = brAPIListDAO; + this.brAPIGermplasmDAO = brAPIGermplasmDAO; + this.sharedStudyService = sharedStudyService; + this.sharedTrialService = sharedTrialService; + } + + @Override + public PendingData process(ImportContext input) { + + List experimentImportRows = ExperimentUtilities.importRowsToExperimentObservations(input.getImportRows()); + Program program = input.getProgram(); + + // Populate pending objects with existing status + Map> observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); + Map> trialByNameNoScope = sharedTrialService.initializeTrialByNameNoScope(program, observationUnitByNameNoScope, experimentImportRows); + Map> studyByNameNoScope = initializeStudyByNameNoScope(program, trialByNameNoScope, observationUnitByNameNoScope, experimentImportRows); + // interesting we're using our data model instead of brapi for locations + Map> locationByName = initializeUniqueLocationNames(program, studyByNameNoScope, experimentImportRows); + Map> obsVarDatasetByName = initializeObsVarDatasetByName(program, trialByNameNoScope, experimentImportRows); + Map> existingGermplasmByGID = initializeExistingGermplasmByGID(program, observationUnitByNameNoScope, experimentImportRows); + + PendingData existing = PendingData.builder() + .observationUnitByNameNoScope(observationUnitByNameNoScope) + .trialByNameNoScope(trialByNameNoScope) + .studyByNameNoScope(studyByNameNoScope) + .locationByName(locationByName) + .obsVarDatasetByName(obsVarDatasetByName) + .existingGermplasmByGID(existingGermplasmByGID) + .build(); + + return existing; + } + + /** + * Initializes the observation units for the given program and experimentImportRows. + * + * @param program The program object + * @param experimentImportRows A list of ExperimentObservation objects + * @return A map of Observation Unit IDs to PendingImportObject objects + * + * @throws InternalServerException + * @throws IllegalStateException + */ + private Map> initializeObservationUnits(Program program, List experimentImportRows) { + Map> observationUnitByName = new HashMap<>(); + + Map rowByObsUnitId = new HashMap<>(); + experimentImportRows.forEach(row -> { + if (StringUtils.isNotBlank(row.getObsUnitID())) { + if(rowByObsUnitId.containsKey(row.getObsUnitID())) { + throw new IllegalStateException("ObsUnitId is repeated: " + row.getObsUnitID()); + } + rowByObsUnitId.put(row.getObsUnitID(), row); + } + }); + + try { + List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); + + // TODO: grab from externalReferences + /* + observationUnitByObsUnitId = existingObsUnits.stream() + .collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, + (BrAPIObservationUnit unit) -> new PendingImportObject<>(unit, false))); + */ + + String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + if (existingObsUnits.size() == rowByObsUnitId.size()) { + existingObsUnits.forEach(brAPIObservationUnit -> { + processAndCacheObservationUnit(brAPIObservationUnit, refSource, program, observationUnitByName, rowByObsUnitId); + + BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) + .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); + + ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); + row.setExpTitle(Utilities.removeProgramKey(brAPIObservationUnit.getTrialName(), program.getKey())); + row.setEnv(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getStudyName(), program.getKey())); + row.setEnvLocation(Utilities.removeProgramKey(brAPIObservationUnit.getLocationName(), program.getKey())); + }); + } else { + List missingIds = new ArrayList<>(rowByObsUnitId.keySet()); + missingIds.removeAll(existingObsUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList())); + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + + return observationUnitByName; + } catch (ApiException e) { + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + /** + * Adds a new map entry to observationUnitByName based on the brAPIObservationUnit passed in and sets the + * expUnitId in the rowsByObsUnitId map. + * + * @param brAPIObservationUnit the BrAPI observation unit object + * @param refSource the reference source + * @param program the program object + * @param observationUnitByName the map of observation units by name (will be modified in place) + * @param rowByObsUnitId the map of rows by observation unit ID (will be modified in place) + * + * @throws InternalServerException + */ + private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservationUnit, String refSource, Program program, + Map> observationUnitByName, + Map rowByObsUnitId) { + BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) + .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); + + ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); + row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); + observationUnitByName.put(createObservationUnitKey(row), + new PendingImportObject<>(ImportObjectState.EXISTING, + brAPIObservationUnit, + UUID.fromString(idRef.getReferenceId()))); + } + + private String createObservationUnitKey(ExperimentObservation importRow) { + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); + } + + private String createObservationUnitKey(String studyName, String obsUnitName) { + return studyName + obsUnitName; + } + + /** + * Initializes studies by name without scope. + * + * @param program The program object. + * @param trialByNameNoScope A map of trial names with their corresponding pending import objects. + * @param experimentImportRows A list of experiment observation objects. + * @return A map of study names with their corresponding pending import objects. + * @throws InternalServerException If there is an error while processing the method. + */ + private Map> initializeStudyByNameNoScope(Program program, + Map> trialByNameNoScope, + Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> studyByName = new HashMap<>(); + if (trialByNameNoScope.size() != 1) { + return studyByName; + } + + try { + initializeStudiesForExistingObservationUnits(program, studyByName, observationUnitByNameNoScope); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing studies", e); + throw new InternalServerException(e.toString(), e); + } + + List existingStudies; + Optional> trial = getTrialPIO(experimentImportRows, trialByNameNoScope); + + try { + if (trial.isEmpty()) { + // TODO: throw ValidatorException and return 422 + } + UUID experimentId = trial.get().getId(); + existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); + for (BrAPIStudy existingStudy : existingStudies) { + sharedStudyService.processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); + } + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing studies: ", e); + throw new InternalServerException(e.toString(), e); + } + + return studyByName; + } + + /** + * Retrieves the PendingImportObject of a BrAPITrial based on the given list of ExperimentObservation and trialByNameNoScope map. + * + * @param experimentImportRows The list of ExperimentObservation objects. + * @param trialByNameNoScope The map of trial names to PendingImportObject of BrAPITrial. + * @return The Optional containing the PendingImportObject of BrAPITrial, or an empty Optional if no matching trial is found. + */ + private Optional> getTrialPIO(List experimentImportRows, + Map> trialByNameNoScope) { + Optional expTitle = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID()) && StringUtils.isNotBlank(row.getExpTitle())) + .map(ExperimentObservation::getExpTitle) + .findFirst(); + + if (expTitle.isEmpty() && trialByNameNoScope.keySet().stream().findFirst().isEmpty()) { + return Optional.empty(); + } + if(expTitle.isEmpty()) { + expTitle = trialByNameNoScope.keySet().stream().findFirst(); + } + + return Optional.ofNullable(trialByNameNoScope.get(expTitle.get())); + } + + + private void initializeStudiesForExistingObservationUnits( + Program program, + Map> studyByName, + Map> observationUnitByNameNoScope + ) throws Exception { + Set studyDbIds = observationUnitByNameNoScope.values() + .stream() + .map(pio -> pio.getBrAPIObject() + .getStudyDbId()) + .collect(Collectors.toSet()); + + List studies = sharedStudyService.fetchStudiesByDbId(studyDbIds, program); + for (BrAPIStudy study : studies) { + sharedStudyService.processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); + } + } + + /** + * Initializes unique location names for a program. + * + * @param program The program object. + * @param studyByNameNoScope A map of study names and corresponding BrAPI study objects. + * @param experimentImportRows A list of experiment observation objects for import. + * @return A map of location names and their corresponding pending import objects. + * @throws InternalServerException If there is an error fetching locations. + */ + private Map> initializeUniqueLocationNames(Program program, + Map> studyByNameNoScope, + List experimentImportRows) { + Map> locationByName = new HashMap<>(); + + List existingLocations = new ArrayList<>(); + if(studyByNameNoScope.size() > 0) { + Set locationDbIds = studyByNameNoScope.values() + .stream() + .map(study -> study.getBrAPIObject() + .getLocationDbId()) + .collect(Collectors.toSet()); + try { + existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueLocationNames = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getEnvLocation) + .distinct() + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + try { + existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); + return locationByName; + } + + /** + * Initializes observation variable dataset by name. + * + * @param program The program associated with the dataset. + * @param trialByNameNoScope The map of trials identified by name without scope. + * @param experimentImportRows The list of experiment observation rows. + * @return The map of observation variable dataset indexed by name. + * + * @throws InternalServerException + */ + private Map> initializeObsVarDatasetByName(Program program, + Map> trialByNameNoScope, + List experimentImportRows) { + Map> obsVarDatasetByName = new HashMap<>(); + + Optional> trialPIO = getTrialPIO(experimentImportRows, trialByNameNoScope); + + if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String datasetId = trialPIO.get().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + try { + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(datasetId)); + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("existing dataset summary not returned from brapi server"); + } + BrAPIListDetails dataSetDetails = brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult(); + processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + return obsVarDatasetByName; + } + + /** + * Process and cache an object of type BrAPIListDetails. + * + * @param existingList The existing list to be processed and cached + * @param obsVarDatasetByName A map of ObsVarDatasets indexed by name (will be modified in place) + * + * @throws IllegalStateException + */ + private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map> obsVarDatasetByName) { + BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for list (dbid): " + existingList.getListDbId())); + obsVarDatasetByName.put(existingList.getListName(), + new PendingImportObject<>(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceId()))); + } + + /** + * Initializes existing germplasm objects by germplasm ID (GID). + * + * @param program The program object. + * @param observationUnitByNameNoScope A map of observation unit objects by name. + * @param experimentImportRows A list of experiment observation objects. + * @return A map of existing germplasm objects by germplasm ID. + * + * @throws InternalServerException + */ + private Map> initializeExistingGermplasmByGID(Program program, + Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> existingGermplasmByGID = new HashMap<>(); + + List existingGermplasms = new ArrayList<>(); + if(observationUnitByNameNoScope.size() > 0) { + Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + try { + existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueGermplasmGIDs = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getGid) + .distinct() + .collect(Collectors.toList()); + + try { + existingGermplasms.addAll(getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + existingGermplasms.forEach(existingGermplasm -> { + BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); + }); + return existingGermplasmByGID; + } + + /** + * Retrieves a list of germplasm with the specified accession numbers. + * + * @param germplasmAccessionNumbers The list of accession numbers to search for. + * @param programId The ID of the program. + * @return An ArrayList of BrAPIGermplasm objects that match the accession numbers. + * @throws ApiException if there is an error retrieving the germplasm. + */ + private ArrayList getGermplasmByAccessionNumber( + List germplasmAccessionNumbers, + UUID programId) throws ApiException { + List germplasmList = brAPIGermplasmDAO.getGermplasm(programId); + ArrayList resultGermplasm = new ArrayList<>(); + // Search for accession number matches + for (BrAPIGermplasm germplasm : germplasmList) { + for (String accessionNumber : germplasmAccessionNumbers) { + if (germplasm.getAccessionNumber() + .equals(accessionNumber)) { + resultGermplasm.add(germplasm); + break; + } + } + } + return resultGermplasm; + } + +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java new file mode 100644 index 000000000..67127c103 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java @@ -0,0 +1,15 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; + +import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; + +public class ProcessStep implements ProcessingStep { + + @Override + public ProcessedData process(PendingData input) { + + // TODO: implement + return new ProcessedData(); + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java new file mode 100644 index 000000000..80af3e2a2 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java @@ -0,0 +1,120 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import io.micronaut.context.annotation.Property; +import io.reactivex.functions.Function; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPISeason; +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +@Singleton +@Slf4j +public class SharedStudyService { + + private final BrAPISeasonDAO brAPISeasonDAO; + private final BrAPIStudyDAO brAPIStudyDAO; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public SharedStudyService(BrAPISeasonDAO brAPISeasonDAO, + BrAPIStudyDAO brAPIStudyDAO) { + this.brAPISeasonDAO = brAPISeasonDAO; + this.brAPIStudyDAO = brAPIStudyDAO; + } + + // TODO: used by both workflows + public PendingImportObject processAndCacheStudy( + BrAPIStudy existingStudy, + Program program, + Function getterFunction, + Map> studyMap) throws Exception { + PendingImportObject pendingStudy; + BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); + // map season dbid to year + String seasonDbId = existingStudy.getSeasons().get(0); // It is assumed that the study has only one season + if(StringUtils.isNotBlank(seasonDbId)) { + String seasonYear = seasonDbIdToYear(seasonDbId, program.getId()); + existingStudy.setSeasons(Collections.singletonList(seasonYear)); + } + pendingStudy = new PendingImportObject<>( + ImportObjectState.EXISTING, + (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), + UUID.fromString(xref.getReferenceId()) + ); + studyMap.put( + Utilities.removeProgramKeyAndUnknownAdditionalData(getterFunction.apply(existingStudy), program.getKey()), + pendingStudy + ); + return pendingStudy; + } + + // TODO: used by both workflows + private String seasonDbIdToYear(String seasonDbId, UUID programId) { + String year = null; + // TODO: add season objects to redis cache then just extract year from those + // removing this for now here + //if (this.seasonDbIdToYearCache.containsKey(seasonDbId)) { // get it from cache if possible + // year = this.seasonDbIdToYearCache.get(seasonDbId); + //} else { + year = seasonDbIdToYearFromDatabase(seasonDbId, programId); + // this.seasonDbIdToYearCache.put(seasonDbId, year); + //} + return year; + } + + // TODO: used by both workflows + private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { + BrAPISeason season = null; + try { + season = this.brAPISeasonDAO.getSeasonById(seasonDbId, programId); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + } + Integer yearInt = (season == null) ? null : season.getYear(); + return (yearInt == null) ? "" : yearInt.toString(); + } + + // TODO: used by both worflows + /** + * Fetches a list of BrAPI studies by their study database IDs for a given program. + * + * This method queries the BrAPIStudyDAO to retrieve studies based on the provided study database IDs and the program. + * It ensures that all requested study database IDs are found in the result set, throwing an IllegalStateException if any are missing. + * + * @param studyDbIds a Set of Strings representing the study database IDs to fetch + * @param program the Program object representing the program context in which to fetch studies + * @return a List of BrAPIStudy objects matching the provided study database IDs + * + * @throws ApiException if there is an issue fetching the studies + * @throws IllegalStateException if any requested study database IDs are not found in the result set + */ + public List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { + List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); + if (studies.size() != studyDbIds.size()) { + List missingIds = new ArrayList<>(studyDbIds); + missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); + throw new IllegalStateException( + "Study not found for studyDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + return studies; + } + +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java new file mode 100644 index 000000000..3ed7ca7a6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java @@ -0,0 +1,180 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +@Singleton +@Slf4j +public class SharedTrialService { + private final BrAPITrialDAO brAPITrialDAO; + + private final SharedStudyService studyService; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public SharedTrialService(BrAPITrialDAO brAPITrialDAO, + SharedStudyService studyService) { + this.brAPITrialDAO = brAPITrialDAO; + this.studyService = studyService; + } + + // TODO: also used in other workflow + /** + * Initializes trials for existing observation units. + * + * @param program The program object. + * @param observationUnitByNameNoScope A map containing observation units by name (without scope). + * @param trialByName A map containing trials by name. (will be modified in place) + * + */ + public void initializeTrialsForExistingObservationUnits(Program program, + Map> observationUnitByNameNoScope, + Map> trialByName) { + if(observationUnitByNameNoScope.size() > 0) { + Set trialDbIds = new HashSet<>(); + Set studyDbIds = new HashSet<>(); + + observationUnitByNameNoScope.values() + .forEach(pio -> { + BrAPIObservationUnit existingOu = pio.getBrAPIObject(); + if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { + throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); + } + + if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { + trialDbIds.add(existingOu.getTrialDbId()); + } else { + studyDbIds.add(existingOu.getStudyDbId()); + } + }); + + //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId + if(!studyDbIds.isEmpty()) { + try { + trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, program)); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + try { + List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); + if (trials.size() != trialDbIds.size()) { + List missingIds = new ArrayList<>(trialDbIds); + missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); + throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + + trials.forEach(trial -> processAndCacheTrial(trial, program, trialByName)); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + } + + /** + * Fetches trial DbIds for the given study DbIds by using the BrAPI studies API. + * + * @param studyDbIds The set of study DbIds for which to fetch trial DbIds. + * @param program The program associated with the studies. + * @return A set of trial DbIds corresponding to the provided study DbIds. + * @throws ApiException If there was an error while fetching the studies or if a study does not have a trial DbId. + * @throws IllegalStateException If the trial DbId is not set for an existing study. + */ + private Set fetchTrialDbidsForStudies(Set studyDbIds, Program program) throws ApiException { + Set trialDbIds = new HashSet<>(); + List studies = studyService.fetchStudiesByDbId(studyDbIds, program); + studies.forEach(study -> { + if (StringUtils.isBlank(study.getTrialDbId())) { + throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); + } + trialDbIds.add(study.getTrialDbId()); + }); + + return trialDbIds; + } + + /** + * This method processes an existing trial, retrieves the experiment ID from the trial's external references, + * and caches the trial with the corresponding experiment ID in a map. + * + * @param existingTrial The existing BrAPITrial object to be processed and cached. + * @param program The Program object associated with the trial. + * @param trialByNameNoScope The map to cache the trial by its name without program scope. (will be modified in place) + * + * @throws InternalServerException + */ + private void processAndCacheTrial( + BrAPITrial existingTrial, + Program program, + Map> trialByNameNoScope) { + + //get TrialId from existingTrial + BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) + .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); + UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); + + trialByNameNoScope.put( + Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), + new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); + } + + /** + * Initializes trials by name without scope for the given program. + * + * @param program the program to initialize trials for + * @param observationUnitByNameNoScope a map of observation units by name without scope + * @param experimentImportRows a list of experiment observation rows + * @return a map of trials by name with pending import objects + * + * @throws InternalServerException + */ + public Map> initializeTrialByNameNoScope(Program program, Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> trialByName = new HashMap<>(); + + initializeTrialsForExistingObservationUnits(program, observationUnitByNameNoScope, trialByName); + + List uniqueTrialNames = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID())) + .map(ExperimentObservation::getExpTitle) + .distinct() + .collect(Collectors.toList()); + try { + brAPITrialDAO.getTrialsByName(uniqueTrialNames, program).forEach(existingTrial -> + processAndCacheTrial(existingTrial, program, trialByName) + ); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + return trialByName; + } + +} \ No newline at end of file From 777b7e2a3edd53f91302344bca81dee5a353d2eb Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 20 May 2024 15:31:31 -0400 Subject: [PATCH 13/50] Added file headers and dynamic column parser --- .../services/pipeline/ProcessingStep.java | 1 - .../experiment/DynamicColumnParser.java | 64 +++++++++++++++++++ .../experiment/ExperimentUtilities.java | 18 ++++++ .../experiment/create/model/PendingData.java | 16 +++++ .../steps/GetExistingProcessingStep.java | 16 +++++ .../create/workflow/steps/ProcessStep.java | 23 +++++++ .../services/SharedStudyService.java | 16 +++++ .../services/SharedTrialService.java | 16 +++++ 8 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java index ca16792db..fb7f5e514 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java @@ -15,7 +15,6 @@ * limitations under the License. */ - package org.breedinginsight.brapps.importer.services.pipeline; public interface ProcessingStep { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java new file mode 100644 index 000000000..6b250facc --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java @@ -0,0 +1,64 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors.experiment; + +import lombok.Getter; +import tech.tablesaw.api.Table; +import tech.tablesaw.columns.Column; + +import java.util.ArrayList; +import java.util.List; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.TIMESTAMP_PREFIX; + +public class DynamicColumnParser { + + /** + * Parses dynamic columns from a table and separates them into phenotype and timestamp columns. + * + * @param data The table containing the dynamic columns. + * @param dynamicColumnNames An array of dynamic column names to be parsed. + * @return A DynamicColumnParseResult object containing the parsed phenotype and timestamp columns. + */ + public static DynamicColumnParseResult parse(Table data, String[] dynamicColumnNames) { + List> dynamicCols = data.columns(dynamicColumnNames); + List> phenotypeCols = new ArrayList<>(); + List> timestampCols = new ArrayList<>(); + + for (Column dynamicCol : dynamicCols) { + if (dynamicCol.name().startsWith(TIMESTAMP_PREFIX)) { + timestampCols.add(dynamicCol); + } else { + phenotypeCols.add(dynamicCol); + } + } + + return new DynamicColumnParseResult(phenotypeCols, timestampCols); + } + + @Getter + public static class DynamicColumnParseResult { + private final List> phenotypeCols; + private final List> timestampCols; + + public DynamicColumnParseResult(List> phenotypeCols, List> timestampCols) { + this.phenotypeCols = phenotypeCols; + this.timestampCols = timestampCols; + } + + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index ed685449d..f6cb96490 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -1,3 +1,19 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.breedinginsight.brapps.importer.services.processors.experiment; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; @@ -9,6 +25,8 @@ public class ExperimentUtilities { public static final CharSequence COMMA_DELIMITER = ","; + public static final String TIMESTAMP_PREFIX = "TS:"; + public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; public static List importRowsToExperimentObservations(List importRows) { return importRows.stream() diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java index 7d6fc74b9..213411078 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -1,3 +1,19 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; import lombok.*; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java index 10afb13a8..5bba6fd52 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java @@ -1,3 +1,19 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; import io.micronaut.context.annotation.Property; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java index 67127c103..8560f6f84 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java @@ -1,3 +1,19 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; @@ -6,9 +22,16 @@ public class ProcessStep implements ProcessingStep { + public ProcessStep() { + + } + @Override public ProcessedData process(PendingData input) { + + + // TODO: implement return new ProcessedData(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java index 80af3e2a2..0e9aef883 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java @@ -1,3 +1,19 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.breedinginsight.brapps.importer.services.processors.experiment.services; import io.micronaut.context.annotation.Property; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java index 3ed7ca7a6..d3d5efd1f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java @@ -1,3 +1,19 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.breedinginsight.brapps.importer.services.processors.experiment.services; import io.micronaut.context.annotation.Property; From 5564ca1e97a7509d0ef6edc353462471ee61ee47 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 22 May 2024 18:17:05 -0400 Subject: [PATCH 14/50] create micronaut composite pattern classes for experiment import workflow --- .../model/imports/ImportServiceContext.java | 1 + .../ExperimentImportService.java | 27 ++++----- .../importer/model/workflow/Action.java | 29 +++++++++ .../model/workflow/ImportWorkflow.java | 12 ++++ .../model/workflow/ImportWorkflowResult.java | 33 +++++++++++ .../importer/services/FileImportService.java | 10 ++-- .../experiment/ExperimentImportWorkflow.java | 59 +++++++++++++++++++ .../AppendOverwriteObservationWorkflow.java | 53 +++++++++++++++++ .../workflow/NewExperimentWorkflow.java | 54 +++++++++++++++++ .../workflow/NewEnvironmentWorkflow.java | 53 +++++++++++++++++ 10 files changed, 309 insertions(+), 22 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java index 45393f67c..c5a479e90 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -32,6 +32,7 @@ @AllArgsConstructor @NoArgsConstructor public class ImportServiceContext { + private String action; private Workflow workflow; private List brAPIImports; private Table data; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index c6f68b251..de7d54961 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -23,9 +23,11 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import tech.tablesaw.api.Table; @@ -34,6 +36,7 @@ import javax.inject.Provider; import javax.inject.Singleton; import java.util.List; +import java.util.Optional; @Singleton @Slf4j @@ -43,12 +46,16 @@ public class ExperimentImportService implements BrAPIImportService { private final Provider experimentProcessorProvider; private final Provider processorManagerProvider; + private final ExperimentImportWorkflow workflow; @Inject - public ExperimentImportService(Provider experimentProcessorProvider, Provider processorManagerProvider) + public ExperimentImportService(Provider experimentProcessorProvider, + Provider processorManagerProvider, + ExperimentImportWorkflow workflow) { this.experimentProcessorProvider = experimentProcessorProvider; this.processorManagerProvider = processorManagerProvider; + this.workflow = workflow; } @Override @@ -70,23 +77,11 @@ public String getMissingColumnMsg(String columnName) { public ImportPreviewResponse process(ImportServiceContext context) throws Exception { - ImportPreviewResponse response = null; - List processors = List.of(experimentProcessorProvider.get()); - - if (context.getWorkflow() != null) { - log.info("Workflow: " + context.getWorkflow().getName()); + if (!context.getAction().isEmpty()) { + log.info("Workflow: " + context.getAction()); } - // TODO: change to calling workflow process instead of processor manager - response = processorManagerProvider.get().process(context.getBrAPIImports(), - processors, - context.getData(), - context.getProgram(), - context.getUpload(), - context.getUser(), - context.isCommit()); - return response; - + return workflow.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java new file mode 100644 index 000000000..65c3d0cbb --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java @@ -0,0 +1,29 @@ +///* +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; + +@Getter +@Setter +@ToString +@AllArgsConstructor +public class Action { + private String name; + private int order; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java new file mode 100644 index 000000000..c5dfc1957 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java @@ -0,0 +1,12 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +import io.micronaut.core.order.Ordered; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; + +import java.util.Optional; + +@FunctionalInterface +public interface ImportWorkflow extends Ordered { + Optional process(ImportServiceContext context); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java new file mode 100644 index 000000000..2e61a6d46 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java @@ -0,0 +1,33 @@ +///* +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; + +import java.util.Optional; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +public class ImportWorkflowResult { + private Action action; + private Optional importPreviewResponse; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index ef6c5f884..59438c696 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -435,14 +435,12 @@ private void processFile(UUID workflowId, List finalBrAPIImportList // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { - Workflow workflow = null; - if (workflowId != null) { - Optional optionalWorkflow = workflowFactory.getWorkflow(workflowId); - workflow = optionalWorkflow.orElse(null); - } + + final Workflow[] workflow = {null}; + Optional.ofNullable(workflowId).ifPresent(id -> workflow[0] = workflowFactory.getWorkflow(id).orElse(null)); ImportServiceContext context = ImportServiceContext.builder() - .workflow(workflow) + .workflow(workflow[0]) .brAPIImports(finalBrAPIImportList) .data(data) .program(program) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java new file mode 100644 index 000000000..e8a7d45da --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java @@ -0,0 +1,59 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment; + +import io.micronaut.context.annotation.Primary; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; + +import javax.inject.Singleton; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +@Primary +@Singleton +public class ExperimentImportWorkflow implements ImportWorkflow { + private final List workflows; + + public ExperimentImportWorkflow(List workflows) { + this.workflows = workflows; + } + + @Override + public Optional process(ImportServiceContext context) { + return workflows.stream() + .map(workflow->workflow.process(context)) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst(); + } + public List getWorkflows() { + return workflows.stream() + .map(workflow->workflow.process(null)) + .filter(Optional::isPresent) + .map(Optional::get) + .map(result->result.getAction()) + .collect(Collectors.toList()); + } + + public enum ImportAction { + APPEND_OVERWRITE("append-overwrite-observation"), + NEW_OBSERVATION("new-observation"), + APPEND_ENVIRONMENT("append-environment"); + + private String urlFragment; + + ImportAction(String urlFragment) { + this.urlFragment = urlFragment; + } + + public String getUrlFragment() { + return urlFragment; + } + + public boolean isEqual(String value) { + return Optional.ofNullable(urlFragment.equals(value)).orElse(false); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java new file mode 100644 index 000000000..ca5e932ef --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java @@ -0,0 +1,53 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; + +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; + +import javax.inject.Singleton; +import java.util.Optional; + +@Singleton +public class AppendOverwriteObservationWorkflow implements ImportWorkflow { + private final ExperimentImportWorkflow.ImportAction action; + + public AppendOverwriteObservationWorkflow(){ + this.action = ExperimentImportWorkflow.ImportAction.APPEND_OVERWRITE; + } + + @Override + public Optional process(ImportServiceContext context) { + // Workflow processing the context + Action workflow = new Action(action.getUrlFragment(), this.getOrder()); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .action(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless appending or overwriting observation data + if (context != null && !action.isEqual(context.getAction())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; + } + + @Override + public int getOrder() { + return 2; + } + + public ExperimentImportWorkflow.ImportAction getAction() { + return this.action; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java new file mode 100644 index 000000000..c29fe9fd5 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java @@ -0,0 +1,54 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; + +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; + +import javax.inject.Singleton; +import java.util.Optional; + +@Singleton +public class NewExperimentWorkflow implements ImportWorkflow { + private final ExperimentImportWorkflow.ImportAction action; + + public NewExperimentWorkflow(){ + this.action = ExperimentImportWorkflow.ImportAction.NEW_OBSERVATION; + } + + @Override + public Optional process(ImportServiceContext context) { + // Workflow processing the context + Action workflow = new Action(action.getUrlFragment(), this.getOrder()); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .action(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless creating a new experiment + if (context != null && !action.isEqual(context.getAction())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; + } + + @Override + public int getOrder() { + return 1; + } + + public ExperimentImportWorkflow.ImportAction getAction() { + return this.action; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java new file mode 100644 index 000000000..57ee9790e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java @@ -0,0 +1,53 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; + +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; + +import javax.inject.Singleton; +import java.util.Optional; + +@Singleton +public class NewEnvironmentWorkflow implements ImportWorkflow { + private final ExperimentImportWorkflow.ImportAction action; + + public NewEnvironmentWorkflow(){ + this.action = ExperimentImportWorkflow.ImportAction.APPEND_ENVIRONMENT; + } + + @Override + public Optional process(ImportServiceContext context) { + // Workflow processing the context + Action workflow = new Action(action.getUrlFragment(), this.getOrder()); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .action(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless appending a new environment + if (context != null && !action.isEqual(context.getAction())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; + } + + @Override + public int getOrder() { + return 3; + } + + public ExperimentImportWorkflow.ImportAction getAction() { + return this.action; + } +} From f726611cef1f5f3b3d82f76fd9b0c3ad5e8c8a01 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 22 May 2024 18:52:39 -0400 Subject: [PATCH 15/50] create FileImportService method for GET workflows --- .../importer/controllers/ImportController.java | 13 ++++++++++--- .../importer/model/imports/BrAPIImportService.java | 4 ++++ .../ExperimentImportService.java | 5 +++++ .../brapps/importer/services/FileImportService.java | 9 +++++++-- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index 5c9ac0ca1..e9e9612e1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -37,6 +37,7 @@ import org.breedinginsight.api.model.v1.response.metadata.StatusCode; import org.breedinginsight.api.v1.controller.metadata.AddMetadata; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; +import org.breedinginsight.brapps.importer.model.workflow.Action; import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; import org.breedinginsight.brapps.importer.services.ImportConfigManager; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; @@ -214,16 +215,22 @@ public HttpResponse>> getSystemMappings(@Nu @Produces(MediaType.APPLICATION_JSON) @AddMetadata @Secured(SecurityRule.IS_ANONYMOUS) - public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { + public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { - List workflows = fileImportService.getWorkflowsForSystemMapping(mappingId); + List workflows = null; + try { + workflows = fileImportService.getWorkflowsForSystemMapping(mappingId); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } List metadataStatus = new ArrayList<>(); metadataStatus.add(new Status(StatusCode.INFO, "Successful Query")); Pagination pagination = new Pagination(workflows.size(), workflows.size(), 1, 0); Metadata metadata = new Metadata(pagination, metadataStatus); - Response> response = new Response(metadata, new DataResponse<>(workflows)); + Response> response = new Response(metadata, new DataResponse<>(workflows)); return HttpResponse.ok(response); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index d06454c30..997155d93 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -18,10 +18,14 @@ package org.breedinginsight.brapps.importer.model.imports; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.Action; + +import java.util.List; public interface BrAPIImportService { String getImportTypeId(); BrAPIImport getImportClass(); + List getWorkflows(); default String getInvalidIntegerMsg(String columnName) { return String.format("Column name \"%s\" must be integer type, but non-integer type provided.", columnName); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index de7d54961..760464d93 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -23,6 +23,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.Action; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; @@ -72,6 +73,10 @@ public String getImportTypeId() { public String getMissingColumnMsg(String columnName) { return "Column heading does not match template or ontology"; } + @Override + public List getWorkflows() { + return workflow.getWorkflows(); + } @Override public ImportPreviewResponse process(ImportServiceContext context) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 59438c696..7d9ebc0c6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -43,6 +43,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; +import org.breedinginsight.brapps.importer.model.workflow.Action; import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.workflow.WorkflowFactory; @@ -584,8 +585,12 @@ public List getSystemMappingByName(String name) { return importMappings; } - public List getWorkflowsForSystemMapping(UUID mappingId) { - return importMappingWorkflowDAO.getWorkflowsByImportMappingId(mappingId); + public List getWorkflowsForSystemMapping(UUID mappingId) throws DoesNotExistException { + ImportMapping mappingConfig = importMappingDAO.getMapping(mappingId) + .orElseThrow(() -> new DoesNotExistException("Cannot find mapping config associated with upload.")); + BrAPIImportService importService = configManager.getImportServiceById(mappingConfig.getImportTypeId()) + .orElseThrow(() -> new DoesNotExistException("Config with that id does not exist")); + return importService.getWorkflows(); } From f6952dd78f45955706e5adbe25eed24ab7b641e2 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 22 May 2024 19:22:18 -0400 Subject: [PATCH 16/50] delete unused classes --- .../controllers/UploadController.java | 12 +-- .../daos/ImportMappingWorkflowDAO.java | 68 --------------- .../importer/services/FileImportService.java | 24 ++---- .../services/workflow/WorkflowFactory.java | 83 ------------------- 4 files changed, 11 insertions(+), 176 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java index 053f1dc3c..5ee6eb062 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java @@ -158,15 +158,15 @@ public HttpResponse> previewData(@PathVariable UUID pro } } - @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/preview") + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflow}/data/{uploadId}/preview") @Produces(MediaType.APPLICATION_JSON) @AddMetadata @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) public HttpResponse> previewData(@PathVariable UUID programId, @PathVariable UUID mappingId, - @PathVariable UUID workflowId, @PathVariable UUID uploadId) { + @PathVariable String workflow, @PathVariable UUID uploadId) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, null, false); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflow, actingUser, null, false); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { @@ -184,16 +184,16 @@ public HttpResponse> previewData(@PathVariable UUID pro } } - @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/commit") + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflow}/data/{uploadId}/commit") @Produces(MediaType.APPLICATION_JSON) @AddMetadata @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) public HttpResponse> commitData(@PathVariable UUID programId, @PathVariable UUID mappingId, - @PathVariable UUID workflowId, @PathVariable UUID uploadId, + @PathVariable String workflow, @PathVariable UUID uploadId, @Body @Nullable Map userInput) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, userInput, true); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflow, actingUser, userInput, true); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java deleted file mode 100644 index 1b60bda45..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.daos; - -import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; -import org.breedinginsight.dao.db.tables.daos.ImporterMappingWorkflowDao; -import org.jooq.Configuration; -import org.jooq.DSLContext; - -import javax.inject.Inject; -import java.util.List; -import java.util.Optional; -import java.util.UUID; - -import static org.breedinginsight.dao.db.Tables.*; - -public class ImportMappingWorkflowDAO extends ImporterMappingWorkflowDao { - - private DSLContext dsl; - - @Inject - public ImportMappingWorkflowDAO(Configuration config, DSLContext dsl) { - super(config); - this.dsl = dsl; - } - - /** - * Retrieves a list of ImportMappingWorkflow objects associated with the given mappingId. They are ordered by - * position for proper ordering on the front end. - * - * @param mappingId The UUID of the mapping to retrieve the workflows for. - * @return A list of ImportMappingWorkflow objects. - */ - public List getWorkflowsByImportMappingId(UUID mappingId) { - return dsl.select() - .from(IMPORTER_MAPPING_WORKFLOW) - .where(IMPORTER_MAPPING_WORKFLOW.MAPPING_ID.eq(mappingId)) - .orderBy(IMPORTER_MAPPING_WORKFLOW.POSITION.asc()) - .fetch(ImportMappingWorkflow::parseSQLRecord); - } - - /** - * Retrieves a workflow by its ID. - * - * @param workflowId The ID of the workflow to retrieve. - * @return An Optional containing the ImportMappingWorkflow if found, otherwise an empty Optional. - */ - public Optional getWorkflowById(UUID workflowId) { - return Optional.ofNullable(fetchOneById(workflowId)) - .map(ImportMappingWorkflow::new); - } - -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 7d9ebc0c6..ffaa68614 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.micronaut.http.HttpStatus; -import io.micronaut.http.annotation.PathVariable; import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.multipart.CompletedFileUpload; import io.micronaut.http.server.exceptions.InternalServerException; @@ -33,7 +32,6 @@ import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.daos.ImportMappingProgramDAO; -import org.breedinginsight.brapps.importer.daos.ImportMappingWorkflowDAO; import org.breedinginsight.brapps.importer.model.ImportProgress; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; @@ -44,12 +42,8 @@ import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; -import org.breedinginsight.brapps.importer.services.workflow.WorkflowFactory; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingProgramEntity; -import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import org.breedinginsight.services.ProgramService; @@ -88,14 +82,12 @@ public class FileImportService { private final ImportDAO importDAO; private final DSLContext dsl; private final ImportMappingProgramDAO importMappingProgramDAO; - private final ImportMappingWorkflowDAO importMappingWorkflowDAO; - private final WorkflowFactory workflowFactory; @Inject FileImportService(ProgramUserService programUserService, ProgramService programService, MimeTypeParser mimeTypeParser, ImportMappingDAO importMappingDAO, ObjectMapper objectMapper, MappingManager mappingManager, ImportConfigManager configManager, ImportDAO importDAO, DSLContext dsl, ImportMappingProgramDAO importMappingProgramDAO, - ImportMappingWorkflowDAO importMappingWorkflowDAO, WorkflowFactory workflowFactory, UserService userService) { + UserService userService) { this.programUserService = programUserService; this.programService = programService; this.mimeTypeParser = mimeTypeParser; @@ -107,8 +99,6 @@ public class FileImportService { this.dsl = dsl; this.importMappingProgramDAO = importMappingProgramDAO; this.userService = userService; - this.importMappingWorkflowDAO = importMappingWorkflowDAO; - this.workflowFactory = workflowFactory; } public List getAllImportTypeConfigs() { @@ -334,7 +324,7 @@ public ImportResponse uploadData(UUID programId, UUID mappingId, AuthenticatedUs return response; } - public ImportResponse updateUpload(UUID programId, UUID uploadId, UUID workflowId, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws + public ImportResponse updateUpload(UUID programId, UUID uploadId, String workflow, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws DoesNotExistException, UnprocessableEntityException, AuthorizationException { Program program = validateRequest(programId, actingUser); @@ -384,7 +374,7 @@ public ImportResponse updateUpload(UUID programId, UUID uploadId, UUID workflowI } else { brAPIImportList = mappingManager.map(mappingConfig, data); } - processFile(workflowId, brAPIImportList, data, program, upload, user, commit, importService, actingUser); + processFile(workflow, brAPIImportList, data, program, upload, user, commit, importService, actingUser); } catch (UnprocessableEntityException e) { log.error(e.getMessage(), e); ImportProgress progress = upload.getProgress(); @@ -430,18 +420,14 @@ public ImportUpload setDynamicColumns(ImportUpload newUpload, Table data, Import return newUpload; } - private void processFile(UUID workflowId, List finalBrAPIImportList, Table data, Program program, + private void processFile(String workflow, List finalBrAPIImportList, Table data, Program program, ImportUpload upload, User user, Boolean commit, BrAPIImportService importService, AuthenticatedUser actingUser) { // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { - - final Workflow[] workflow = {null}; - Optional.ofNullable(workflowId).ifPresent(id -> workflow[0] = workflowFactory.getWorkflow(id).orElse(null)); - ImportServiceContext context = ImportServiceContext.builder() - .workflow(workflow[0]) + .action(workflow) .brAPIImports(finalBrAPIImportList) .data(data) .program(program) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java deleted file mode 100644 index 17d6fe64e..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.services.workflow; - -import io.micronaut.context.ApplicationContext; -import io.micronaut.context.annotation.Factory; -import io.micronaut.context.exceptions.NoSuchBeanException; -import io.micronaut.inject.qualifiers.Qualifiers; -import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.daos.ImportMappingWorkflowDAO; -import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; - -import javax.inject.Inject; -import java.util.Optional; -import java.util.UUID; - -@Factory -@Slf4j -public class WorkflowFactory { - - private final ImportMappingWorkflowDAO importMappingWorkflowDAO; - private final ApplicationContext applicationContext; - - @Inject - public WorkflowFactory(ImportMappingWorkflowDAO importMappingWorkflowDAO, - ApplicationContext applicationContext) { - this.importMappingWorkflowDAO = importMappingWorkflowDAO; - this.applicationContext = applicationContext; - } - - /** - * Produces the appropriate workflow instance based on the import context - * - * @param context the import context - * @return an Optional containing the workflow if id is not null, otherwise an empty Optional - * - * @throws IllegalStateException - * @throws NoSuchBeanException - */ - public Optional getWorkflow(UUID workflowId) { - - if (workflowId != null) { - // construct workflow from db record - Optional workflowOptional = importMappingWorkflowDAO.getWorkflowById(workflowId); - - ImportMappingWorkflow importMappingworkflow = workflowOptional.orElseThrow(() -> { - String msg = "Must have record in db for workflowId"; - log.error(msg); - return new IllegalStateException(msg); - }); - - // newer versions of micronaut have fancier ways to do this using annotations with provider but as - // far as I can tell it's not available in 2.5 - Workflow workflow; - try { - workflow = applicationContext.getBean(Workflow.class, Qualifiers.byName(importMappingworkflow.getBean())); - } catch (NoSuchBeanException e) { - log.error("Could not find workflow class implementation for bean: " + importMappingworkflow.getBean()); - throw e; - } - - return Optional.of(workflow); - } - - return Optional.empty(); - } -} From 5e31aa6f363f7b1767b49fe59b79c793cd10a373 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 22 May 2024 20:33:43 -0400 Subject: [PATCH 17/50] clean up and renaming --- .../controllers/ImportController.java | 9 +-- .../model/imports/BrAPIImportService.java | 4 +- .../model/imports/ImportServiceContext.java | 5 +- .../ExperimentImportService.java | 22 ++--- .../importer/model/workflow/Action.java | 29 ------- .../model/workflow/ImportWorkflow.java | 35 ++++++-- .../model/workflow/ImportWorkflowResult.java | 2 +- .../importer/model/workflow/Workflow.java | 37 ++------- .../importer/services/FileImportService.java | 6 +- ...tWorkflow.java => ExperimentWorkflow.java} | 26 +++--- .../AppendOverwriteObservationWorkflow.java | 53 ------------ .../AppendOverwritePhenotypesWorkflow.java | 81 ++++++++++--------- .../workflow/CreateNewExperimentWorkflow.java | 79 +++++++++--------- .../workflow/NewExperimentWorkflow.java | 54 ------------- .../CreateNewEnvironmentWorkflow.java | 81 ++++++++++--------- .../workflow/NewEnvironmentWorkflow.java | 53 ------------ .../V1.22.0__add_experiment_workflows.sql | 51 ------------ 17 files changed, 195 insertions(+), 432 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ExperimentImportWorkflow.java => ExperimentWorkflow.java} (61%) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java delete mode 100644 src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index e9e9612e1..1a16dadd7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -37,8 +37,7 @@ import org.breedinginsight.api.model.v1.response.metadata.StatusCode; import org.breedinginsight.api.v1.controller.metadata.AddMetadata; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; -import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.ImportConfigManager; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; import org.breedinginsight.brapps.importer.services.FileImportService; @@ -215,9 +214,9 @@ public HttpResponse>> getSystemMappings(@Nu @Produces(MediaType.APPLICATION_JSON) @AddMetadata @Secured(SecurityRule.IS_ANONYMOUS) - public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { + public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { - List workflows = null; + List workflows = null; try { workflows = fileImportService.getWorkflowsForSystemMapping(mappingId); } catch (DoesNotExistException e) { @@ -230,7 +229,7 @@ public HttpResponse>> getWorkflowsForSystemMapping Pagination pagination = new Pagination(workflows.size(), workflows.size(), 1, 0); Metadata metadata = new Metadata(pagination, metadataStatus); - Response> response = new Response(metadata, new DataResponse<>(workflows)); + Response> response = new Response(metadata, new DataResponse<>(workflows)); return HttpResponse.ok(response); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index 997155d93..3f25d99e6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -18,14 +18,14 @@ package org.breedinginsight.brapps.importer.model.imports; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import java.util.List; public interface BrAPIImportService { String getImportTypeId(); BrAPIImport getImportClass(); - List getWorkflows(); + List getWorkflows(); default String getInvalidIntegerMsg(String columnName) { return String.format("Column name \"%s\" must be integer type, but non-integer type provided.", columnName); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java index c5a479e90..4d052cd33 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -19,10 +19,10 @@ import lombok.*; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import tech.tablesaw.api.Table; + import java.util.List; @Getter @@ -32,8 +32,7 @@ @AllArgsConstructor @NoArgsConstructor public class ImportServiceContext { - private String action; - private Workflow workflow; + private String workflow; private List brAPIImports; private Table data; private Program program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 760464d93..df60e9782 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -18,26 +18,18 @@ package org.breedinginsight.brapps.importer.model.imports.experimentObservation; import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; -import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import tech.tablesaw.api.Table; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; import java.util.List; -import java.util.Optional; @Singleton @Slf4j @@ -47,12 +39,12 @@ public class ExperimentImportService implements BrAPIImportService { private final Provider experimentProcessorProvider; private final Provider processorManagerProvider; - private final ExperimentImportWorkflow workflow; + private final ExperimentWorkflow workflow; @Inject public ExperimentImportService(Provider experimentProcessorProvider, Provider processorManagerProvider, - ExperimentImportWorkflow workflow) + ExperimentWorkflow workflow) { this.experimentProcessorProvider = experimentProcessorProvider; this.processorManagerProvider = processorManagerProvider; @@ -74,7 +66,7 @@ public String getMissingColumnMsg(String columnName) { return "Column heading does not match template or ontology"; } @Override - public List getWorkflows() { + public List getWorkflows() { return workflow.getWorkflows(); } @@ -82,8 +74,8 @@ public List getWorkflows() { public ImportPreviewResponse process(ImportServiceContext context) throws Exception { - if (!context.getAction().isEmpty()) { - log.info("Workflow: " + context.getAction()); + if (!context.getWorkflow().isEmpty()) { + log.info("Workflow: " + context.getWorkflow()); } return workflow.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java deleted file mode 100644 index 65c3d0cbb..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java +++ /dev/null @@ -1,29 +0,0 @@ -///* -// * See the NOTICE file distributed with this work for additional information -// * regarding copyright ownership. -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ - -package org.breedinginsight.brapps.importer.model.workflow; - -import lombok.*; - -@Getter -@Setter -@ToString -@AllArgsConstructor -public class Action { - private String name; - private int order; -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java index c5dfc1957..b26a886bd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java @@ -1,12 +1,31 @@ -package org.breedinginsight.brapps.importer.model.workflow; +///* +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ -import io.micronaut.core.order.Ordered; -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +package org.breedinginsight.brapps.importer.model.workflow; -import java.util.Optional; +import lombok.*; -@FunctionalInterface -public interface ImportWorkflow extends Ordered { - Optional process(ImportServiceContext context); +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +public class ImportWorkflow { + private String urlFragment; + private String displayName; + private int order; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java index 2e61a6d46..9304c1ce2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java @@ -28,6 +28,6 @@ @ToString @AllArgsConstructor public class ImportWorkflowResult { - private Action action; + private ImportWorkflow workflow; private Optional importPreviewResponse; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java index 9ea0e7bf9..8bc19120e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -1,36 +1,11 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.breedinginsight.brapps.importer.model.workflow; -public interface Workflow { +import io.micronaut.core.order.Ordered; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; - /** - * Processes the given import context and returns the processed data. - * - * @param context the import context containing the necessary data for processing - * @return the processed data - */ - ProcessedData process(ImportContext context); +import java.util.Optional; - /** - * Retrieves the name of the Workflow for logging display purposes. - * - * @return the name of the Workflow - */ - String getName(); +@FunctionalInterface +public interface Workflow extends Ordered { + Optional process(ImportServiceContext context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index ffaa68614..6a1914ab9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -41,7 +41,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; -import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingProgramEntity; import org.breedinginsight.model.Program; @@ -427,7 +427,7 @@ private void processFile(String workflow, List finalBrAPIImportList CompletableFuture.supplyAsync(() -> { try { ImportServiceContext context = ImportServiceContext.builder() - .action(workflow) + .workflow(workflow) .brAPIImports(finalBrAPIImportList) .data(data) .program(program) @@ -571,7 +571,7 @@ public List getSystemMappingByName(String name) { return importMappings; } - public List getWorkflowsForSystemMapping(UUID mappingId) throws DoesNotExistException { + public List getWorkflowsForSystemMapping(UUID mappingId) throws DoesNotExistException { ImportMapping mappingConfig = importMappingDAO.getMapping(mappingId) .orElseThrow(() -> new DoesNotExistException("Cannot find mapping config associated with upload.")); BrAPIImportService importService = configManager.getImportServiceById(mappingConfig.getImportTypeId()) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java similarity index 61% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java index e8a7d45da..04bbc4fc1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java @@ -2,8 +2,8 @@ import io.micronaut.context.annotation.Primary; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.workflow.Action; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import javax.inject.Singleton; @@ -13,10 +13,10 @@ @Primary @Singleton -public class ExperimentImportWorkflow implements ImportWorkflow { - private final List workflows; +public class ExperimentWorkflow implements Workflow { + private final List workflows; - public ExperimentImportWorkflow(List workflows) { + public ExperimentWorkflow(List workflows) { this.workflows = workflows; } @@ -28,29 +28,33 @@ public Optional process(ImportServiceContext context) { .map(Optional::get) .findFirst(); } - public List getWorkflows() { + public List getWorkflows() { return workflows.stream() .map(workflow->workflow.process(null)) .filter(Optional::isPresent) .map(Optional::get) - .map(result->result.getAction()) + .map(result->result.getWorkflow()) .collect(Collectors.toList()); } - public enum ImportAction { - APPEND_OVERWRITE("append-overwrite-observation"), - NEW_OBSERVATION("new-observation"), - APPEND_ENVIRONMENT("append-environment"); + public enum Workflow { + NEW_OBSERVATION("new-experiment","Create new experiment"), + APPEND_OVERWRITE("append-dataset", "Append experimental dataset"), + APPEND_ENVIRONMENT("append-environment", "Create new experimental environment"); private String urlFragment; + private String displayName; + + Workflow(String urlFragment, String displayName) { - ImportAction(String urlFragment) { this.urlFragment = urlFragment; + this.displayName = displayName; } public String getUrlFragment() { return urlFragment; } + public String getDisplayName() { return displayName; } public boolean isEqual(String value) { return Optional.ofNullable(urlFragment.equals(value)).orElse(false); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java deleted file mode 100644 index ca5e932ef..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; - -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; - -import javax.inject.Singleton; -import java.util.Optional; - -@Singleton -public class AppendOverwriteObservationWorkflow implements ImportWorkflow { - private final ExperimentImportWorkflow.ImportAction action; - - public AppendOverwriteObservationWorkflow(){ - this.action = ExperimentImportWorkflow.ImportAction.APPEND_OVERWRITE; - } - - @Override - public Optional process(ImportServiceContext context) { - // Workflow processing the context - Action workflow = new Action(action.getUrlFragment(), this.getOrder()); - - // No-preview result - Optional result = Optional.of(ImportWorkflowResult.builder() - .action(workflow) - .importPreviewResponse(Optional.empty()) - .build()); - - // Skip this workflow unless appending or overwriting observation data - if (context != null && !action.isEqual(context.getAction())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return result; - } - - // Start processing the import... - return result; - } - - @Override - public int getOrder() { - return 2; - } - - public ExperimentImportWorkflow.ImportAction getAction() { - return this.action; - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index 21df19be6..44fb575bf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -1,50 +1,55 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; -import io.micronaut.context.annotation.Prototype; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import lombok.Getter; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; -import javax.inject.Named; - -/** - * This class represents a workflow for appending and overwriting phenotypes. The bean name must match the appropriate - * bean column value in the import_mapping_workflow db table - */ +import javax.inject.Singleton; +import java.util.Optional; -@Prototype -@Named("AppendOverwritePhenotypesWorkflow") +@Getter +@Singleton public class AppendOverwritePhenotypesWorkflow implements Workflow { + private final ExperimentWorkflow.Workflow workflow; + + public AppendOverwritePhenotypesWorkflow(){ + this.workflow = ExperimentWorkflow.Workflow.APPEND_OVERWRITE; + } + @Override - public ProcessedData process(ImportContext context) { - // TODO - return null; + public Optional process(ImportServiceContext context) { + // Workflow processing the context + ImportWorkflow workflow = ImportWorkflow.builder() + .urlFragment(getWorkflow().getUrlFragment()) + .displayName(getWorkflow().getDisplayName()) + .build(); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .workflow(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless appending or overwriting observation data + if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; } - /** - * Retrieves the name of the workflow. This is used for logging display purposes. - * - * @return the name of the workflow - */ @Override - public String getName() { - return "AppendOverwritePhenotypesWorkflow"; + public int getOrder() { + return 2; } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 6f6ab661a..3c593ac74 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -1,50 +1,55 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; -import io.micronaut.context.annotation.Prototype; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import lombok.Getter; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; -import javax.inject.Named; +import javax.inject.Singleton; +import java.util.Optional; -/** - * This class represents a workflow for creating a new experiment. The bean name must match the appropriate bean column - * value in the import_mapping_workflow db table - */ -@Prototype -@Named("CreateNewExperimentWorkflow") +@Getter +@Singleton public class CreateNewExperimentWorkflow implements Workflow { + private final ExperimentWorkflow.Workflow workflow; + + public CreateNewExperimentWorkflow(){ + this.workflow = ExperimentWorkflow.Workflow.NEW_OBSERVATION; + } @Override - public ProcessedData process(ImportContext context) { - // TODO - return null; + public Optional process(ImportServiceContext context) { + // Workflow processing the context + ImportWorkflow workflow = ImportWorkflow.builder() + .urlFragment(getWorkflow().getUrlFragment()) + .displayName(getWorkflow().getDisplayName()) + .build(); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .workflow(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless creating a new experiment + if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; } - /** - * Retrieves the name of the workflow. This is used for logging display purposes. - * - * @return the name of the workflow - */ @Override - public String getName() { - return "CreateNewExperimentWorkflow"; + public int getOrder() { + return 1; } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java deleted file mode 100644 index c29fe9fd5..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; - -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; - -import javax.inject.Singleton; -import java.util.Optional; - -@Singleton -public class NewExperimentWorkflow implements ImportWorkflow { - private final ExperimentImportWorkflow.ImportAction action; - - public NewExperimentWorkflow(){ - this.action = ExperimentImportWorkflow.ImportAction.NEW_OBSERVATION; - } - - @Override - public Optional process(ImportServiceContext context) { - // Workflow processing the context - Action workflow = new Action(action.getUrlFragment(), this.getOrder()); - - // No-preview result - Optional result = Optional.of(ImportWorkflowResult.builder() - .action(workflow) - .importPreviewResponse(Optional.empty()) - .build()); - - // Skip this workflow unless creating a new experiment - if (context != null && !action.isEqual(context.getAction())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return result; - } - - // Start processing the import... - return result; - } - - @Override - public int getOrder() { - return 1; - } - - public ExperimentImportWorkflow.ImportAction getAction() { - return this.action; - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java index 5776ab59b..fe0b90c1f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java @@ -1,50 +1,55 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; -import io.micronaut.context.annotation.Prototype; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import lombok.Getter; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; -import javax.inject.Named; - -/** - * This class represents a workflow for adding new environments to an existing experiment. The bean name must match - * the appropriate bean column value in the import_mapping_workflow db table - */ +import javax.inject.Singleton; +import java.util.Optional; -@Prototype -@Named("CreateNewEnvironmentWorkflow") +@Getter +@Singleton public class CreateNewEnvironmentWorkflow implements Workflow { + private final ExperimentWorkflow.Workflow workflow; + + public CreateNewEnvironmentWorkflow(){ + this.workflow = ExperimentWorkflow.Workflow.APPEND_ENVIRONMENT; + } + @Override - public ProcessedData process(ImportContext context) { - // TODO - return null; + public Optional process(ImportServiceContext context) { + // Workflow processing the context + ImportWorkflow workflow = ImportWorkflow.builder() + .urlFragment(getWorkflow().getUrlFragment()) + .displayName(getWorkflow().getDisplayName()) + .build(); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .workflow(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless appending a new environment + if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; } - /** - * Retrieves the name of the workflow. This is used for logging display purposes. - * - * @return the name of the workflow - */ @Override - public String getName() { - return "CreateNewEnvironmentWorkflow"; + public int getOrder() { + return 3; } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java deleted file mode 100644 index 57ee9790e..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; - -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; - -import javax.inject.Singleton; -import java.util.Optional; - -@Singleton -public class NewEnvironmentWorkflow implements ImportWorkflow { - private final ExperimentImportWorkflow.ImportAction action; - - public NewEnvironmentWorkflow(){ - this.action = ExperimentImportWorkflow.ImportAction.APPEND_ENVIRONMENT; - } - - @Override - public Optional process(ImportServiceContext context) { - // Workflow processing the context - Action workflow = new Action(action.getUrlFragment(), this.getOrder()); - - // No-preview result - Optional result = Optional.of(ImportWorkflowResult.builder() - .action(workflow) - .importPreviewResponse(Optional.empty()) - .build()); - - // Skip this workflow unless appending a new environment - if (context != null && !action.isEqual(context.getAction())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return result; - } - - // Start processing the import... - return result; - } - - @Override - public int getOrder() { - return 3; - } - - public ExperimentImportWorkflow.ImportAction getAction() { - return this.action; - } -} diff --git a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql deleted file mode 100644 index 2c9d4d547..000000000 --- a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql +++ /dev/null @@ -1,51 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - Table maps workflows to import mappings and provides required configuration options - - mapping_id - link to importer_mapping that this provides a workflow for - name - name that will be displayed on front end - bean - must match @Named("") annotation on Workflow class - position - for ordering records explicitly, wanted for front end default option and order - */ -CREATE TABLE importer_mapping_workflow -( - like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, - mapping_id UUID NOT NULL, - name TEXT NOT NULL, - bean TEXT NOT NULL, - position INTEGER NOT NULL -); - -ALTER TABLE importer_mapping_workflow - ADD FOREIGN KEY (mapping_id) REFERENCES importer_mapping (id); - -DO -$$ -DECLARE - exp_mapping_id UUID; -BEGIN - exp_mapping_id := (SELECT id FROM importer_mapping WHERE name = 'ExperimentsTemplateMap'); - -INSERT INTO public.importer_mapping_workflow (mapping_id, name, bean, position) -VALUES - (exp_mapping_id, 'Create new experiment', 'CreateNewExperimentWorkflow', 0), - (exp_mapping_id, 'Append experimental dataset', 'AppendOverwritePhenotypesWorkflow', 1), - (exp_mapping_id, 'Create new experimental environment', 'CreateNewEnvironmentWorkflow', 2); -END -$$; \ No newline at end of file From 708793dae1c11e78ece376f1a5fdd3813fcc821d Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 22 May 2024 20:38:09 -0400 Subject: [PATCH 18/50] add overrides to import services --- .../model/imports/germplasm/GermplasmImportService.java | 6 ++++++ .../model/imports/sample/SampleSubmissionImportService.java | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index ce52f483f..3c3a65b4d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -23,6 +23,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.GermplasmProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; @@ -57,6 +58,11 @@ public GermplasmImport getImportClass() { return new GermplasmImport(); } + @Override + public List getWorkflows() { + return null; + } + @Override public String getImportTypeId() { return IMPORT_TYPE_ID; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java index 2c8e8b7b5..55848b0a1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java @@ -23,6 +23,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.brapps.importer.services.processors.SampleSubmissionProcessor; @@ -59,6 +60,11 @@ public BrAPIImport getImportClass() { return new SampleSubmissionImport(); } + @Override + public List getWorkflows() { + return null; + } + @Override public ImportPreviewResponse process(ImportServiceContext context) throws Exception { List processors = List.of(sampleProcessorProvider.get()); From 68bdacb282b335af1874db9ede5ad421de4806ae Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 23 May 2024 06:33:16 -0400 Subject: [PATCH 19/50] create ExperimentWorkflow interface that extends Workflow interface --- .../ExperimentImportService.java | 12 ++++++------ .../importer/model/workflow/ExperimentWorkflow.java | 6 ++++++ .../brapps/importer/model/workflow/Workflow.java | 11 ----------- ...orkflow.java => ExperimentWorkflowNavigator.java} | 8 ++++---- .../workflow/AppendOverwritePhenotypesWorkflow.java | 10 +++++----- .../create/workflow/CreateNewExperimentWorkflow.java | 10 +++++----- .../workflow/CreateNewEnvironmentWorkflow.java | 10 +++++----- 7 files changed, 31 insertions(+), 36 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ExperimentWorkflow.java => ExperimentWorkflowNavigator.java} (85%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index df60e9782..c6d191bf0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -24,7 +24,7 @@ import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import javax.inject.Inject; import javax.inject.Provider; @@ -39,16 +39,16 @@ public class ExperimentImportService implements BrAPIImportService { private final Provider experimentProcessorProvider; private final Provider processorManagerProvider; - private final ExperimentWorkflow workflow; + private final ExperimentWorkflowNavigator workflowNavigator; @Inject public ExperimentImportService(Provider experimentProcessorProvider, Provider processorManagerProvider, - ExperimentWorkflow workflow) + ExperimentWorkflowNavigator workflowNavigator) { this.experimentProcessorProvider = experimentProcessorProvider; this.processorManagerProvider = processorManagerProvider; - this.workflow = workflow; + this.workflowNavigator = workflowNavigator; } @Override @@ -67,7 +67,7 @@ public String getMissingColumnMsg(String columnName) { } @Override public List getWorkflows() { - return workflow.getWorkflows(); + return workflowNavigator.getWorkflows(); } @Override @@ -78,7 +78,7 @@ public ImportPreviewResponse process(ImportServiceContext context) log.info("Workflow: " + context.getWorkflow()); } - return workflow.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); + return workflowNavigator.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java new file mode 100644 index 000000000..c34794bab --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java @@ -0,0 +1,6 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +@FunctionalInterface +public interface ExperimentWorkflow extends Workflow { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java deleted file mode 100644 index 8bc19120e..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.breedinginsight.brapps.importer.model.workflow; - -import io.micronaut.core.order.Ordered; -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; - -import java.util.Optional; - -@FunctionalInterface -public interface Workflow extends Ordered { - Optional process(ImportServiceContext context); -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java similarity index 85% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index 04bbc4fc1..bcf12d543 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -3,7 +3,7 @@ import io.micronaut.context.annotation.Primary; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import javax.inject.Singleton; @@ -13,10 +13,10 @@ @Primary @Singleton -public class ExperimentWorkflow implements Workflow { - private final List workflows; +public class ExperimentWorkflowNavigator implements ExperimentWorkflow { + private final List workflows; - public ExperimentWorkflow(List workflows) { + public ExperimentWorkflowNavigator(List workflows) { this.workflows = workflows; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index 44fb575bf..e64a2f88a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -4,19 +4,19 @@ import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import javax.inject.Singleton; import java.util.Optional; @Getter @Singleton -public class AppendOverwritePhenotypesWorkflow implements Workflow { - private final ExperimentWorkflow.Workflow workflow; +public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow { + private final ExperimentWorkflowNavigator.Workflow workflow; public AppendOverwritePhenotypesWorkflow(){ - this.workflow = ExperimentWorkflow.Workflow.APPEND_OVERWRITE; + this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE; } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 3c593ac74..022727c5a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -4,19 +4,19 @@ import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import javax.inject.Singleton; import java.util.Optional; @Getter @Singleton -public class CreateNewExperimentWorkflow implements Workflow { - private final ExperimentWorkflow.Workflow workflow; +public class CreateNewExperimentWorkflow implements ExperimentWorkflow { + private final ExperimentWorkflowNavigator.Workflow workflow; public CreateNewExperimentWorkflow(){ - this.workflow = ExperimentWorkflow.Workflow.NEW_OBSERVATION; + this.workflow = ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION; } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java index fe0b90c1f..a5695e7f8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java @@ -4,19 +4,19 @@ import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import javax.inject.Singleton; import java.util.Optional; @Getter @Singleton -public class CreateNewEnvironmentWorkflow implements Workflow { - private final ExperimentWorkflow.Workflow workflow; +public class CreateNewEnvironmentWorkflow implements ExperimentWorkflow { + private final ExperimentWorkflowNavigator.Workflow workflow; public CreateNewEnvironmentWorkflow(){ - this.workflow = ExperimentWorkflow.Workflow.APPEND_ENVIRONMENT; + this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_ENVIRONMENT; } @Override From 0e022f12675712ad57574d4b7a27e2e5cac6700f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 23 May 2024 06:48:39 -0400 Subject: [PATCH 20/50] create interfaces for germplasm and sample workflows --- .../model/workflow/ImportMappingWorkflow.java | 56 ------------------- 1 file changed, 56 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java deleted file mode 100644 index 8dc5800bc..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.model.workflow; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; -import lombok.experimental.Accessors; -import lombok.experimental.SuperBuilder; - -import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; -import org.jooq.Record; - -import static org.breedinginsight.dao.db.tables.ImporterMappingWorkflowTable.IMPORTER_MAPPING_WORKFLOW; - -@Getter -@Setter -@Accessors(chain=true) -@ToString -@NoArgsConstructor -@SuperBuilder() -public class ImportMappingWorkflow extends ImporterMappingWorkflowEntity { - - - public ImportMappingWorkflow(ImporterMappingWorkflowEntity importMappingWorkflowEntity) { - this.setId(importMappingWorkflowEntity.getId()); - this.setName(importMappingWorkflowEntity.getName()); - this.setBean(importMappingWorkflowEntity.getBean()); - this.setPosition(importMappingWorkflowEntity.getPosition()); - } - public static ImportMappingWorkflow parseSQLRecord(Record record) { - - return ImportMappingWorkflow.builder() - .id(record.getValue(IMPORTER_MAPPING_WORKFLOW.ID)) - .name(record.getValue(IMPORTER_MAPPING_WORKFLOW.NAME)) - .bean(record.getValue(IMPORTER_MAPPING_WORKFLOW.BEAN)) - .position(record.getValue(IMPORTER_MAPPING_WORKFLOW.POSITION)) - .build(); - } -} From f2a1034abf7ab01abf784255e4dd09f60c7c77b0 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 23 May 2024 08:14:51 -0400 Subject: [PATCH 21/50] create interfaces --- .../importer/model/workflow/GermplasmWorkflow.java | 6 ++++++ .../model/workflow/SampleSubmissionWorkflow.java | 6 ++++++ .../brapps/importer/model/workflow/Workflow.java | 11 +++++++++++ 3 files changed, 23 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java new file mode 100644 index 000000000..f26342d3b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java @@ -0,0 +1,6 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +@FunctionalInterface +public interface GermplasmWorkflow extends Workflow { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java new file mode 100644 index 000000000..19be9bdcc --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java @@ -0,0 +1,6 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +@FunctionalInterface +public interface SampleSubmissionWorkflow extends Workflow { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java new file mode 100644 index 000000000..8bc19120e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -0,0 +1,11 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +import io.micronaut.core.order.Ordered; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; + +import java.util.Optional; + +@FunctionalInterface +public interface Workflow extends Ordered { + Optional process(ImportServiceContext context); +} From 402eb5de2dd58f1fa8076db344a88e150a5ed390 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 31 May 2024 10:40:55 -0400 Subject: [PATCH 22/50] create DomainImportService abstract class --- .../model/imports/DomainImportService.java | 72 +++++++++++++++++++ .../ExperimentImportService.java | 32 +++------ .../importer/model/workflow/Workflow.java | 6 ++ 3 files changed, 86 insertions(+), 24 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java new file mode 100644 index 000000000..5f89de64a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -0,0 +1,72 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.imports; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; +import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.List; + +@Singleton +@Slf4j +public abstract class DomainImportService implements BrAPIImportService { + + private final Provider experimentProcessorProvider; + private final Provider processorManagerProvider; + private final Workflow workflowNavigator; + + @Inject + public DomainImportService(Provider experimentProcessorProvider, + Provider processorManagerProvider) + { + this.experimentProcessorProvider = experimentProcessorProvider; + this.processorManagerProvider = processorManagerProvider; + this.workflowNavigator = getNavigator(); + } + + protected abstract Workflow getNavigator(); + @Override + public String getMissingColumnMsg(String columnName) { + return "Column heading does not match template or ontology"; + } + @Override + public List getWorkflows() { + return workflowNavigator.getWorkflows(); + } + + @Override + public ImportPreviewResponse process(ImportServiceContext context) + throws Exception { + + if (!context.getWorkflow().isEmpty()) { + log.info("Workflow: " + context.getWorkflow()); + } + + return workflowNavigator.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); + } +} + diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index c6d191bf0..60fe4bb3b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -19,9 +19,11 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.DomainImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; @@ -33,12 +35,9 @@ @Singleton @Slf4j -public class ExperimentImportService implements BrAPIImportService { +public class ExperimentImportService extends DomainImportService { private final String IMPORT_TYPE_ID = "ExperimentImport"; - - private final Provider experimentProcessorProvider; - private final Provider processorManagerProvider; private final ExperimentWorkflowNavigator workflowNavigator; @Inject @@ -46,10 +45,13 @@ public ExperimentImportService(Provider experimentProcessor Provider processorManagerProvider, ExperimentWorkflowNavigator workflowNavigator) { - this.experimentProcessorProvider = experimentProcessorProvider; - this.processorManagerProvider = processorManagerProvider; + super(experimentProcessorProvider, processorManagerProvider); this.workflowNavigator = workflowNavigator; } + @Override + public Workflow getNavigator() { + return this.workflowNavigator; + } @Override public ExperimentObservation getImportClass() { @@ -61,24 +63,6 @@ public String getImportTypeId() { return IMPORT_TYPE_ID; } - @Override - public String getMissingColumnMsg(String columnName) { - return "Column heading does not match template or ontology"; - } - @Override - public List getWorkflows() { - return workflowNavigator.getWorkflows(); - } - - @Override - public ImportPreviewResponse process(ImportServiceContext context) - throws Exception { - if (!context.getWorkflow().isEmpty()) { - log.info("Workflow: " + context.getWorkflow()); - } - - return workflowNavigator.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); - } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java index 8bc19120e..17f48ee57 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -3,9 +3,15 @@ import io.micronaut.core.order.Ordered; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; @FunctionalInterface public interface Workflow extends Ordered { Optional process(ImportServiceContext context); + default List getWorkflows() { + // Default implementation for getWorkflows method + return new ArrayList<>(); + } } From c5dd1390e7eba4dbd06f519939c2c7c6885f43e0 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 3 Jun 2024 11:18:57 -0400 Subject: [PATCH 23/50] Shared services work in progress --- .../experiment/ExperimentUtilities.java | 13 + .../create/model/PendingImportObjectData.java | 36 ++ .../create/model/ProcessContext.java | 31 + .../create/model/ProcessedPhenotypeData.java | 37 ++ .../workflow/CreateNewExperimentWorkflow.java | 6 +- .../CreatePendingImportPopulator.java | 67 ++ ...lateExistingPendingImportObjectsStep.java} | 26 +- ...ulateModifiedPendingImportObjectsStep.java | 5 + .../PopulateNewPendingImportObjectsStep.java | 4 + .../create/workflow/steps/ProcessStep.java | 580 +++++++++++++++++- .../PendingImportObjectPopulator.java | 41 ++ .../services/SharedPendingImportService.java | 22 + .../services/SharedPhenotypeService.java | 76 +++ .../services/SharedSeasonService.java | 97 +++ .../services/SharedValidateService.java | 116 ++++ 15 files changed, 1139 insertions(+), 18 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingImportObjectData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessContext.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/{GetExistingProcessingStep.java => PopulateExistingPendingImportObjectsStep.java} (95%) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateModifiedPendingImportObjectsStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPendingImportService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPhenotypeService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedSeasonService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedValidateService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index f6cb96490..9983b1aaa 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -19,6 +19,8 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.*; import java.util.stream.Collectors; @@ -27,10 +29,21 @@ public class ExperimentUtilities { public static final CharSequence COMMA_DELIMITER = ","; public static final String TIMESTAMP_PREFIX = "TS:"; public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; + public static final String MIDNIGHT = "T00:00:00-00:00"; public static List importRowsToExperimentObservations(List importRows) { return importRows.stream() .map(trialImport -> (ExperimentObservation) trialImport) .collect(Collectors.toList()); } + + public static boolean validDateTimeValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingImportObjectData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingImportObjectData.java new file mode 100644 index 000000000..1f26cdf8d --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingImportObjectData.java @@ -0,0 +1,36 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; + +import lombok.*; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class PendingImportObjectData { + private PendingImportObject trialPIO; + private PendingImportObject studyPIO; + private PendingImportObject obsUnitPIO; + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessContext.java new file mode 100644 index 000000000..b66f7207c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessContext.java @@ -0,0 +1,31 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ProcessContext { + private PendingData pendingData; + private ImportContext importContext; +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java new file mode 100644 index 000000000..cd6842476 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java @@ -0,0 +1,37 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; + +import lombok.*; +import org.breedinginsight.model.Trait; +import tech.tablesaw.columns.Column; + +import java.util.List; +import java.util.Map; + +// TODO: move to common higher level location +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ProcessedPhenotypeData { + private Map> timeStampColByPheno; + private List referencedTraits; + private List> phenotypeCols; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index ebfa5469d..4666ac0f1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -22,7 +22,7 @@ import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.pipeline.Pipeline; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.GetExistingProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.PopulateExistingPendingImportObjectsStep; import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.ProcessStep; import javax.inject.Inject; @@ -38,11 +38,11 @@ @Named("CreateNewExperimentWorkflow") public class CreateNewExperimentWorkflow implements Workflow { - private final Provider getExistingStepProvider; + private final Provider getExistingStepProvider; private final Provider processStepProvider; @Inject - public CreateNewExperimentWorkflow(Provider getExistingStepProvider, + public CreateNewExperimentWorkflow(Provider getExistingStepProvider, Provider processStepProvider) { this.getExistingStepProvider = getExistingStepProvider; this.processStepProvider = processStepProvider; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java new file mode 100644 index 000000000..ea839298b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java @@ -0,0 +1,67 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; + +import org.apache.commons.lang3.StringUtils; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.PendingImportObjectPopulator; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import java.math.BigInteger; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; + +public class CreatePendingImportPopulator implements PendingImportObjectPopulator { + + @Override + public PendingImportObject populateTrial(ImportContext importContext, + PendingData pendingData, + ExperimentObservation importRow, + Supplier expNextVal) + throws UnprocessableEntityException { + + PendingImportObject trialPio; + Program program = importContext.getProgram(); + User user = importContext.getUser(); + boolean commit = importContext.isCommit(); + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + + if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { + PendingImportObject envPio; + trialPio = trialByNameNoScope.get(importRow.getExpTitle()); + envPio = studyByNameNoScope.get(importRow.getEnv()); + + // creating new units for existing experiments and environments is not possible + if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && + (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ + throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); + } + } else if (!trialByNameNoScope.isEmpty()) { + throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); + } else { + UUID id = UUID.randomUUID(); + String expSeqValue = null; + if (commit) { + expSeqValue = expNextVal.get().toString(); + } + BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); + trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); + // TODO: move + //trialByNameNoScope.put(importRow.getExpTitle(), trialPio); + } + + return trialPio; + } + + + + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java similarity index 95% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index 5bba6fd52..8ee355dc3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -40,6 +40,7 @@ import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedStudyService; import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedTrialService; import org.breedinginsight.model.Program; @@ -58,7 +59,7 @@ @Prototype @Slf4j -public class GetExistingProcessingStep implements ProcessingStep { +public class PopulateExistingPendingImportObjectsStep implements ProcessingStep { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final BrAPITrialDAO brAPITrialDAO; @@ -73,14 +74,14 @@ public class GetExistingProcessingStep implements ProcessingStep experimentImportRows = ExperimentUtilities.importRowsToExperimentObservations(input.getImportRows()); Program program = input.getProgram(); @@ -115,7 +116,10 @@ public PendingData process(ImportContext input) { .existingGermplasmByGID(existingGermplasmByGID) .build(); - return existing; + return ProcessContext.builder() + .importContext(input) + .pendingData(existing) + .build(); } /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateModifiedPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateModifiedPendingImportObjectsStep.java new file mode 100644 index 000000000..a04d33540 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateModifiedPendingImportObjectsStep.java @@ -0,0 +1,5 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; + +// TODO: think this would be for other workflow +public class PopulateModifiedPendingImportObjectsStep { +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java new file mode 100644 index 000000000..13870fc51 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -0,0 +1,4 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; + +public class PopulateNewPendingImportObjectsStep { +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java index 8560f6f84..b6f6e7796 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java @@ -16,23 +16,595 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingImportObjectData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.PendingImportObjectPopulator; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedPhenotypeService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedSeasonService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedValidateService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.model.User; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; +import org.jooq.DSLContext; +import tech.tablesaw.api.Table; +import org.breedinginsight.model.Trait; +import tech.tablesaw.columns.Column; + +import javax.inject.Inject; +import java.math.BigInteger; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Collectors; -public class ProcessStep implements ProcessingStep { +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.*; - public ProcessStep() { +@Prototype +@Slf4j +public class ProcessStep implements ProcessingStep { + private final SharedValidateService sharedValidateService; + private final SharedSeasonService sharedSeasonService; + private final SharedPhenotypeService sharedPhenotypeService; + private final BrAPIObservationDAO brAPIObservationDAO; + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private final DSLContext dsl; + + @Inject + public ProcessStep(SharedValidateService sharedValidateService, + SharedSeasonService sharedSeasonService, + SharedPhenotypeService sharedPhenotypeService, + BrAPIObservationDAO brAPIObservationDAO, + BrAPIObservationUnitDAO brAPIObservationUnitDAO, + DSLContext dsl) { + this.sharedValidateService = sharedValidateService; + this.sharedSeasonService = sharedSeasonService; + this.sharedPhenotypeService = sharedPhenotypeService; + this.brAPIObservationDAO = brAPIObservationDAO; + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + this.dsl = dsl; } @Override - public ProcessedData process(PendingData input) { + public ProcessedData process(ProcessContext context) { + + Table data = context.getImportContext().getData(); + ImportUpload upload = context.getImportContext().getUpload(); + ImportContext importContext = context.getImportContext(); + ProcessedPhenotypeData phenotypeData = sharedPhenotypeService.extractPhenotypes(importContext); // TODO: implement return new ProcessedData(); } + + +/* + // initNew + private void populatePendingImportObjects(ImportContext importContext, + ProcessedPhenotypeData phenotypeData, + PendingImportObjectPopulator pioPopulator) { + + List importRows = importContext.getImportRows(); + Program program = importContext.getProgram(); + + Supplier expNextVal = getNextExperimentSequenceNumber(program); + Supplier envNextVal = getNextEnvironmentSequenceNumber(program); + + // TODO: handle this + // existingObsByObsHash = fetchExistingObservations(referencedTraits, program); + + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + + populateIndependentVariablePIOsForRow(); + + // ... (Common logic from the original method) + + PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); + + processObservations(importContext, phenotypeData, importRow, rowNum, commit, importRow.getEnvYear(), obsUnitPIO, studyPIO); + } + } +*/ + + /** + * Returns a Supplier that generates the next experiment sequence number based on the given Program. + * + * @param program the Program for which to generate the next experiment sequence number + * @return a Supplier that generates the next experiment sequence number + * @throws HttpStatusException if the program is not properly configured for observation unit import + */ + private Supplier getNextExperimentSequenceNumber(Program program) { + String expSequenceName = program.getExpSequence(); + if (expSequenceName == null) { + log.error(String.format("Program, %s, is missing a value in the exp sequence column.", program.getName())); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for observation unit import"); + } + return () -> dsl.nextval(expSequenceName.toLowerCase()); + } + + /** + * Retrieves the next environment sequence number for a given program. + * + * @param program The program for which to get the next environment sequence number. + * @return A Supplier representing a function that generates the next environment sequence number. + * @throws HttpStatusException If the program is not properly configured for environment import. + */ + private Supplier getNextEnvironmentSequenceNumber(Program program) { + String envSequenceName = program.getEnvSequence(); + if (envSequenceName == null) { + log.error(String.format("Program, %s, is missing a value in the env sequence column.", program.getName())); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for environment import"); + } + return () -> dsl.nextval(envSequenceName.toLowerCase()); + } + + /** + * Populates independent variable PendingImportObjectData for a given row of import data. + * + * @param importContext The import context. + * @param phenotypeData The processed phenotype data. + * @param pendingData The pending data. + * @param importRow The import row. + * @param expNextVal The supplier for generating experiment next value. + * @param envNextVal The supplier for generating environment next value. + * @param pioPopulator The pending import object populator. + * @return The populated independent variable PendingImportObjectData. + * @throws MissingRequiredInfoException If any required information is missing. + * @throws UnprocessableEntityException If the entity is unprocessable. + * @throws ApiException If there is an API exception. + */ + private PendingImportObjectData populateIndependentVariablePIOsForRow(ImportContext importContext, + ProcessedPhenotypeData phenotypeData, + PendingData pendingData, + ExperimentObservation importRow, + Supplier expNextVal, + Supplier envNextVal, + PendingImportObjectPopulator pioPopulator) + throws MissingRequiredInfoException, UnprocessableEntityException, ApiException { + + Program program = importContext.getProgram(); + User user = importContext.getUser(); + boolean commit = importContext.isCommit(); + List referencedTraits = phenotypeData.getReferencedTraits(); + + PendingImportObject trialPIO = null; + try { + trialPIO = pioPopulator.populateTrial(importContext, importRow, expNextVal); + + // moved up a level + if (trialPIO.getState() == ImportObjectState.NEW) { + pendingData.getTrialByNameNoScope().put(importRow.getExpTitle(), trialPIO); + } + } catch (UnprocessableEntityException e) { + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } + + String expSeqValue = null; + if (commit) { + expSeqValue = trialPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString(); + } + + if (commit) { + fetchOrCreateDatasetPIO(importRow, program, referencedTraits); + } + + fetchOrCreateLocationPIO(importRow); + + PendingImportObject studyPIO = fetchOrCreateStudyPIO(program, commit, expSeqValue, importRow, envNextVal); + + String envSeqValue = null; + if (commit) { + envSeqValue = studyPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER) + .getAsString(); + } + + PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); + + return PendingImportObjectData.builder() + .trialPIO(trialPIO) + .studyPIO(studyPIO) + .obsUnitPIO(obsUnitPIO) + .build(); + } + +/* + private void processObservations(ImportContext importContext, ProcessedPhenotypeData phenotypeData, ExperimentObservation importRow, + int rowNum, boolean commit, String studyYear, + PendingImportObject obsUnitPIO, + PendingImportObject studyPIO) + throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { + Program program = importContext.getProgram(); + User user = importContext.getUser(); + List> phenotypeCols = phenotypeData.getPhenotypeCols(); + Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); + List referencedTraits = phenotypeData.getReferencedTraits(); + + for (Column column : phenotypeCols) { + // ... (Logic for processing observations) + } + } + + // TODO: move common code out + private void initNewBrapiData(ImportContext importContext, ProcessedPhenotypeData phenotypeData) + throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { + + Program program = importContext.getProgram(); + List importRows = importContext.getImportRows(); + User user = importContext.getUser(); + boolean commit = importContext.isCommit(); + Map existingObsByObsHash; + + List referencedTraits = phenotypeData.getReferencedTraits(); + List> phenotypeCols = phenotypeData.getPhenotypeCols(); + Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); + + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + + + for (Column column : phenotypeCols) { + //If associated timestamp column, add + String dateTimeValue = null; + if (timeStampColByPheno.containsKey(column.name())) { + dateTimeValue = timeStampColByPheno.get(column.name()).getString(rowNum); + //If no timestamp, set to midnight + if (!dateTimeValue.isBlank() && !validDateTimeValue(dateTimeValue)) { + dateTimeValue += MIDNIGHT; + } + } + + // get the study year either referenced from the observation unit or listed explicitly on the import row + // TODO: handle this different workflows + String studyYear = hasAllReferenceUnitIds ? studyPIO.getBrAPIObject().getSeasons().get(0) : importRow.getEnvYear(); + String seasonDbId = sharedSeasonService.yearToSeasonDbId(studyYear, program.getId()); + fetchOrCreateObservationPIO( + program, + user, + importRow, + column, //column.name() gets phenotype name + rowNum, + dateTimeValue, + commit, + seasonDbId, + obsUnitPIO, + studyPIO, + referencedTraits + ); + } + } + } + + private Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { + Set ouDbIds = new HashSet<>(); + Set variableDbIds = new HashSet<>(); + Map variableNameByDbId = new HashMap<>(); + Map ouNameByDbId = new HashMap<>(); + Map studyNameByDbId = studyByNameNoScope.values() + .stream() + .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) + .map(PendingImportObject::getBrAPIObject) + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); + + studyNameByDbId.keySet().forEach(studyDbId -> { + try { + brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { + if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { + ouDbIds.add(ou.getObservationUnitDbId()); + } + ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + }); + } catch (ApiException e) { + throw new RuntimeException(e); + } + }); + + for (Trait referencedTrait : referencedTraits) { + variableDbIds.add(referencedTrait.getObservationVariableDbId()); + variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); + } + + List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); + + return existingObservations.stream() + .map(obs -> { + String studyName = studyNameByDbId.get(obs.getStudyDbId()); + String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); + String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); + + String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); + + return Map.entry(key, obs); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private PendingImportObject fetchOrCreateTrialPIO( + Program program, + User user, + boolean commit, + ExperimentObservation importRow, + Supplier expNextVal + ) throws UnprocessableEntityException { + PendingImportObject trialPio; + + // use the prior trial if observation unit IDs are supplied + // TODO: handle multiple workflows + if (hasAllReferenceUnitIds) { + trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); + + // otherwise create a new trial, but there can be only one allowed + } else { + if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { + PendingImportObject envPio; + trialPio = trialByNameNoScope.get(importRow.getExpTitle()); + envPio = studyByNameNoScope.get(importRow.getEnv()); + + // creating new units for existing experiments and environments is not possible + if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && + (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ + throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); + } + } else if (!trialByNameNoScope.isEmpty()) { + throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); + } else { + UUID id = UUID.randomUUID(); + String expSeqValue = null; + if (commit) { + expSeqValue = expNextVal.get().toString(); + } + BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); + trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); + trialByNameNoScope.put(importRow.getExpTitle(), trialPio); + } + } + return trialPio; + } + + private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) throws UnprocessableEntityException { + PendingImportObject pio; + + // TODO: multiple workflows + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + String name = String.format("Observation Dataset [%s-%s]", + program.getKey(), + trialPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString()); + if (obsVarDatasetByName.containsKey(name)) { + pio = obsVarDatasetByName.get(name); + } else { + UUID id = UUID.randomUUID(); + BrAPIListDetails newDataset = importRow.constructDatasetDetails( + name, + id, + BRAPI_REFERENCE_SOURCE, + program, + trialPIO.getId().toString()); + pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); + trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); + if (ImportObjectState.EXISTING == trialPIO.getState()) { + trialPIO.setState(ImportObjectState.MUTATED); + } + obsVarDatasetByName.put(name, pio); + } + addObsVarsToDatasetDetails(pio, referencedTraits, program); + } + + private void fetchOrCreateLocationPIO(ExperimentObservation importRow) { + PendingImportObject pio; + // TODO: multiple workflows + String envLocationName = hasAllReferenceUnitIds ? + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName() : importRow.getEnvLocation(); + if (!locationByName.containsKey((importRow.getEnvLocation()))) { + ProgramLocation newLocation = new ProgramLocation(); + newLocation.setName(envLocationName); + pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); + this.locationByName.put(envLocationName, pio); + } + } + + private PendingImportObject fetchOrCreateStudyPIO( + Program program, + boolean commit, + String expSequenceValue, + ExperimentObservation importRow, + Supplier envNextVal + ) throws UnprocessableEntityException { + PendingImportObject pio; + // TODO: multiple workflows + if (hasAllReferenceUnitIds) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), + program.getKey() + ); + pio = studyByNameNoScope.get(studyName); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { + pio = studyByNameNoScope.get(importRow.getEnv()); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else { + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + UUID trialID = trialPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); + newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies + + // It is assumed that the study has only one season, And that the Years and not + // the dbId's are stored in getSeason() list. + String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season + if (commit) { + if(StringUtils.isNotBlank(year)) { + String seasonID = this.yearToSeasonDbId(year, program.getId()); + newStudy.setSeasons(Collections.singletonList(seasonID)); + } + } else { + addYearToStudyAdditionalInfo(program, newStudy, year); + } + + pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); + this.studyByNameNoScope.put(importRow.getEnv(), pio); + } + return pio; + } + + private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { + PendingImportObject pio; + String key = createObservationUnitKey(importRow); + // TODO: multiple workflows + if (hasAllReferenceUnitIds) { + pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); + } else if (observationUnitByNameNoScope.containsKey(key)) { + pio = observationUnitByNameNoScope.get(key); + } else { + String germplasmName = ""; + if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { + germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) + .getBrAPIObject() + .getGermplasmName(); + } + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; + UUID trialID = trialPIO.getId(); + UUID datasetId = null; + if (commit) { + datasetId = UUID.fromString(trialPIO.getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + } + PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); + + // check for existing units if this is an existing study + if (studyPIO.getBrAPIObject().getStudyDbId() != null) { + List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); + List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); + if (matchingOU.isEmpty()) { + throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); + } else { + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); + } + } else { + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); + } + this.observationUnitByNameNoScope.put(key, pio); + } + return pio; + } + + private void fetchOrCreateObservationPIO(Program program, + User user, + ExperimentObservation importRow, + Column column, + Integer rowNum, + String timeStampValue, + boolean commit, + String seasonDbId, + PendingImportObject obsUnitPIO, + PendingImportObject studyPIO, + List referencedTraits) throws ApiException, UnprocessableEntityException { + PendingImportObject pio; + BrAPIObservation newObservation; + String variableName = column.name(); + String value = column.getString(rowNum); + String key; + + // TODO: multiple workflows + if (hasAllReferenceUnitIds) { + String unitName = obsUnitPIO.getBrAPIObject().getObservationUnitName(); + String studyName = studyPIO.getBrAPIObject().getStudyName(); + key = getObservationHash(studyName + unitName, variableName, studyName); + } else { + key = getImportObservationHash(importRow, variableName); + } + + if (existingObsByObsHash.containsKey(key)) { + if (!isObservationMatched(key, value, column, rowNum)){ + + // prior observation with updated value + newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); + if (!isValueMatched(key, value)){ + newObservation.setValue(value); + } else if (!StringUtils.isBlank(timeStampValue) && !isTimestampMatched(key, timeStampValue)) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timeStampValue)); + newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + } + pio = new PendingImportObject<>(ImportObjectState.MUTATED, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); + } else { + + // prior observation + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(existingObsByObsHash.get(key), BrAPIObservation.class, program)); + } + + observationByHash.put(key, pio); + } else if (!this.observationByHash.containsKey(key)){ + + // new observation + // TODO: multiple workflows + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + + UUID trialID = trialPIO.getId(); + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject(), commit, program, user, BRAPI_REFERENCE_SOURCE, trialID, studyID, obsUnitPIO.getId(), id); + //NOTE: Can't parse invalid timestamp value, so have to skip if invalid. + // Validation error should be thrown for offending value, but that doesn't happen until later downstream + if (timeStampValue != null && !timeStampValue.isBlank() && (validDateValue(timeStampValue) || validDateTimeValue(timeStampValue))) { + newObservation.setObservationTimeStamp(OffsetDateTime.parse(timeStampValue)); + } + + newObservation.setStudyDbId(studyPIO.getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations + + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); + this.observationByHash.put(key, pio); + } + } + + */ + } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java new file mode 100644 index 000000000..d189a461d --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java @@ -0,0 +1,41 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import java.math.BigInteger; +import java.util.function.Supplier; + +public interface PendingImportObjectPopulator { + + PendingImportObject populateTrial( + ImportContext importContext, + PendingData pendingData, + ExperimentObservation importRow, + Supplier expNextVal + ) throws UnprocessableEntityException; + + + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPendingImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPendingImportService.java new file mode 100644 index 000000000..a5d091d69 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPendingImportService.java @@ -0,0 +1,22 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +public class SharedPendingImportService { + + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPhenotypeService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPhenotypeService.java new file mode 100644 index 000000000..6b9bd7dc5 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPhenotypeService.java @@ -0,0 +1,76 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.DynamicColumnParser; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import tech.tablesaw.api.Table; +import tech.tablesaw.columns.Column; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.TIMESTAMP_REGEX; + +@Singleton +@Slf4j +public class SharedPhenotypeService { + + private final SharedValidateService sharedValidateService; + + @Inject + public SharedPhenotypeService(SharedValidateService sharedValidateService) { + this.sharedValidateService = sharedValidateService; + } + + /** + * Extracts phenotypes from the import context. + * + * @param importContext The import context containing the data, upload, and program information. + * @return A ProcessedPhenotypeData object with the extracted phenotypes. + */ + public ProcessedPhenotypeData extractPhenotypes(ImportContext importContext) { + Table data = importContext.getData(); + ImportUpload upload = importContext.getUpload(); + Program program = importContext.getProgram(); + + DynamicColumnParser.DynamicColumnParseResult result = DynamicColumnParser.parse(data, upload.getDynamicColumnNames()); + List traits = sharedValidateService.verifyTraits(program.getId(), result); + + Map> timeStampColByPheno = new HashMap<>(); + //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval + for (Column tsColumn : result.getTimestampCols()) { + timeStampColByPheno.put(tsColumn.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), tsColumn); + } + + return ProcessedPhenotypeData.builder() + .referencedTraits(traits) + .phenotypeCols(result.getPhenotypeCols()) + .timeStampColByPheno(timeStampColByPheno) + .build(); + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedSeasonService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedSeasonService.java new file mode 100644 index 000000000..d168a8a16 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedSeasonService.java @@ -0,0 +1,97 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPISeason; +import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@Singleton +@Slf4j +public class SharedSeasonService { + + private final BrAPISeasonDAO brAPISeasonDAO; + + // TODO: move season to actual cache rather than cacheing at application layer + private Map yearToSeasonDbIdCache = new HashMap<>(); + + @Inject + public SharedSeasonService(BrAPISeasonDAO brAPISeasonDAO) { + this.brAPISeasonDAO = brAPISeasonDAO; + } + + /** + * Converts year String to SeasonDbId + *
+ * NOTE: This assumes that the only Season records of interest are ones + * with a blank name or a name that is the same as the year. + * + * @param year The year as a string + * @param programId the program ID. + * @return the DbId of the season-record associated with the year + */ + public String yearToSeasonDbId(String year, UUID programId) { + String dbID = null; + if (yearToSeasonDbIdCache.containsKey(year)) { // get it from cache if possible + dbID = yearToSeasonDbIdCache.get(year); + } else { + dbID = yearToSeasonDbIdFromDatabase(year, programId); + yearToSeasonDbIdCache.put(year, dbID); + } + return dbID; + } + + private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { + BrAPISeason targetSeason = null; + List seasons; + try { + seasons = brAPISeasonDAO.getSeasonsByYear(year, programId); + for (BrAPISeason season : seasons) { + if (null == season.getSeasonName() || season.getSeasonName().isBlank() || season.getSeasonName().equals(year)) { + targetSeason = season; + break; + } + } + if (targetSeason == null) { + BrAPISeason newSeason = new BrAPISeason(); + Integer intYear = null; + if( StringUtils.isNotBlank(year) ){ + intYear = Integer.parseInt(year); + } + newSeason.setYear(intYear); + newSeason.setSeasonName(year); + targetSeason = brAPISeasonDAO.addOneSeason(newSeason, programId); + } + + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + log.error(e.getResponseBody(), e); + } + + return (targetSeason == null) ? null : targetSeason.getSeasonDbId(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedValidateService.java new file mode 100644 index 000000000..f2edfc3f5 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedValidateService.java @@ -0,0 +1,116 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.brapps.importer.services.FileMappingUtil; +import org.breedinginsight.brapps.importer.services.processors.experiment.DynamicColumnParser.DynamicColumnParseResult; +import org.breedinginsight.dao.db.tables.pojos.TraitEntity; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import tech.tablesaw.columns.Column; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.COMMA_DELIMITER; +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.TIMESTAMP_REGEX; + +@Singleton +@Slf4j +public class SharedValidateService { + + private final OntologyService ontologyService; + private final FileMappingUtil fileMappingUtil; + + @Inject + public SharedValidateService(OntologyService ontologyService, FileMappingUtil fileMappingUtil) { + this.ontologyService = ontologyService; + this.fileMappingUtil = fileMappingUtil; + } + + /** + * Verifies traits based on program ID and dynamic column parse result. + * + * @param programId The UUID of the program. + * @param cols The dynamic column parse result object containing phenotype and timestamp columns. + * @return The list of verified traits. + * @throws HttpStatusException If ontology terms are not found or timestamp columns lack corresponding phenotype columns. + */ + public List verifyTraits(UUID programId, DynamicColumnParseResult cols) { + Set varNames = cols.getPhenotypeCols().stream() + .map(Column::name) + .collect(Collectors.toSet()); + Set tsNames = cols.getTimestampCols().stream() + .map(Column::name) + .collect(Collectors.toSet()); + + // filter out just traits specified in file + List filteredTraits = fetchFileTraits(programId, varNames); + + // check that all specified ontology terms were found + if (filteredTraits.size() != varNames.size()) { + Set returnedVarNames = filteredTraits.stream() + .map(TraitEntity::getObservationVariableName) + .collect(Collectors.toSet()); + List differences = varNames.stream() + .filter(var -> !returnedVarNames.contains(var)) + .collect(Collectors.toList()); + //TODO convert this to a ValidationError + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, + "Ontology term(s) not found: " + String.join(COMMA_DELIMITER, differences)); + } + + // Check that each ts column corresponds to a phenotype column + List unmatchedTimestamps = tsNames.stream() + .filter(e -> !(varNames.contains(e.replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY)))) + .collect(Collectors.toList()); + if (unmatchedTimestamps.size() > 0) { + //TODO convert this to a ValidationError + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, + "Timestamp column(s) lack corresponding phenotype column(s): " + String.join(COMMA_DELIMITER, unmatchedTimestamps)); + } + + // sort the verified traits to match the order of the trait columns + List phenotypeColNames = cols.getPhenotypeCols().stream().map(Column::name).collect(Collectors.toList()); + return fileMappingUtil.sortByField(phenotypeColNames, filteredTraits, TraitEntity::getObservationVariableName); + } + + private List fetchFileTraits(UUID programId, Collection varNames) { + try { + Collection upperCaseVarNames = varNames.stream().map(String::toUpperCase).collect(Collectors.toList()); + List traits = ontologyService.getTraitsByProgramId(programId, true); + // filter out just traits specified in file + return traits.stream() + .filter(e -> upperCaseVarNames.contains(e.getObservationVariableName().toUpperCase())) + .collect(Collectors.toList()); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + throw new InternalServerException(e.toString(), e); + } + } +} From 8825462c5c1d460d03c4cd80746b8307cc31215a Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 10 Jun 2024 10:44:22 -0400 Subject: [PATCH 24/50] Renamed common services --- .../importer/services/pipeline/Pipeline.java | 35 ------------------- ...ulateModifiedPendingImportObjectsStep.java | 2 +- .../CommitPendingImportObjectsStep.java} | 11 +++--- ...va => ExperimentPendingImportService.java} | 0 ...e.java => ExperimentPhenotypeService.java} | 0 ...vice.java => ExperimentSeasonService.java} | 0 ...rvice.java => ExperimentStudyService.java} | 0 ...rvice.java => ExperimentTrialService.java} | 0 ...ce.java => ExperimentValidateService.java} | 0 9 files changed, 8 insertions(+), 40 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/pipeline/Pipeline.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{create => append}/workflow/steps/PopulateModifiedPendingImportObjectsStep.java (74%) rename src/main/java/org/breedinginsight/brapps/importer/services/{pipeline/ProcessingStep.java => processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java} (59%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/{SharedPendingImportService.java => ExperimentPendingImportService.java} (100%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/{SharedPhenotypeService.java => ExperimentPhenotypeService.java} (100%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/{SharedSeasonService.java => ExperimentSeasonService.java} (100%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/{SharedStudyService.java => ExperimentStudyService.java} (100%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/{SharedTrialService.java => ExperimentTrialService.java} (100%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/{SharedValidateService.java => ExperimentValidateService.java} (100%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/Pipeline.java b/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/Pipeline.java deleted file mode 100644 index 76c4433ee..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/Pipeline.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.services.pipeline; - -public class Pipeline { - - private final ProcessingStep currentStep; - - public Pipeline(ProcessingStep currentStep) { - this.currentStep = currentStep; - } - - public Pipeline addProcessingStep(ProcessingStep newStep) { - return new Pipeline<>(input -> newStep.process(currentStep.process(input))); - } - - public O execute(I input) { - return currentStep.process(input); - } -} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateModifiedPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java similarity index 74% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateModifiedPendingImportObjectsStep.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java index a04d33540..e05b1cf25 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateModifiedPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java @@ -1,5 +1,5 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; -// TODO: think this would be for other workflow +// TODO: think this would be for other workflow only public class PopulateModifiedPendingImportObjectsStep { } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java similarity index 59% rename from src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index fb7f5e514..02ab60e3e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/pipeline/ProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -14,9 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; -package org.breedinginsight.brapps.importer.services.pipeline; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; -public interface ProcessingStep { - O process(I input); -} \ No newline at end of file +public class CommitPendingImportDataStep implements ProcessingStep { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPendingImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPendingImportService.java similarity index 100% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPendingImportService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPendingImportService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPhenotypeService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPhenotypeService.java similarity index 100% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedPhenotypeService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPhenotypeService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedSeasonService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java similarity index 100% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedSeasonService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentStudyService.java similarity index 100% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedStudyService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentStudyService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java similarity index 100% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedTrialService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java similarity index 100% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/SharedValidateService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java From 2b0256df11a8044c654bafd52d798dce3edf7fe6 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 10 Jun 2024 10:45:36 -0400 Subject: [PATCH 25/50] Simplified workflow step calling --- .../model/workflow/ImportContext.java | 1 + .../model/workflow/ProcessedData.java | 6 + .../importer/model/workflow/Workflow.java | 4 +- .../services/ImportStatusService.java | 4 + .../services/processors/ProcessorData.java | 8 +- .../experiment/ExperimentUtilities.java | 13 + ...ulateModifiedPendingImportObjectsStep.java | 2 +- .../experiment/create/model/PendingData.java | 3 + .../workflow/CreateNewExperimentWorkflow.java | 120 +++- .../CreatePendingImportPopulator.java | 19 +- .../steps/CommitPendingImportObjectsStep.java | 58 +- ...ulateExistingPendingImportObjectsStep.java | 101 ++- .../PopulateNewPendingImportObjectsStep.java | 599 +++++++++++++++++- .../create/workflow/steps/ProcessStep.java | 41 +- .../ExperimentPendingImportService.java | 2 +- .../services/ExperimentPhenotypeService.java | 10 +- .../services/ExperimentSeasonService.java | 4 +- .../services/ExperimentStudyService.java | 6 +- .../services/ExperimentTrialService.java | 8 +- .../services/ExperimentValidateService.java | 4 +- .../PendingImportObjectPopulator.java | 8 +- 21 files changed, 927 insertions(+), 94 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java index 121ef6b19..109c300cc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java @@ -39,6 +39,7 @@ public class ImportContext { private UUID workflowId; private ImportUpload upload; private List importRows; + // TODO: move this out potentially private Map mappedBrAPIImport; private Table data; private Program program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java index f9f8196c2..73302bdb9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -18,8 +18,13 @@ package org.breedinginsight.brapps.importer.model.workflow; import lombok.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; +import java.util.ArrayList; +import java.util.List; import java.util.Map; @Data @@ -27,4 +32,5 @@ @NoArgsConstructor public class ProcessedData { private Map statistics; + private Map mappedBrAPIImport; } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java index 9ea0e7bf9..8b2165933 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -17,6 +17,8 @@ package org.breedinginsight.brapps.importer.model.workflow; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; + public interface Workflow { /** @@ -25,7 +27,7 @@ public interface Workflow { * @param context the import context containing the necessary data for processing * @return the processed data */ - ProcessedData process(ImportContext context); + ImportPreviewResponse process(ImportContext context); /** * Retrieves the name of the Workflow for logging display purposes. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/ImportStatusService.java b/src/main/java/org/breedinginsight/brapps/importer/services/ImportStatusService.java index e1dcefc63..d69934ed0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/ImportStatusService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/ImportStatusService.java @@ -42,17 +42,20 @@ public ImportStatusService(ImportDAO importDAO, ObjectMapper objMapper) { } public void updateMessage(ImportUpload upload, String message) { + log.debug(message); upload.getProgress().setMessage(message); importDAO.update(upload); } public void startUpload(ImportUpload upload, long numberObjects, String message) { + log.debug(message); upload.getProgress().setTotal(numberObjects); upload.getProgress().setMessage(message); importDAO.update(upload); } public void finishUpload(ImportUpload upload, long numberObjects, String message) { + log.debug(message); // Update progress to reflect final finished and inProgress counts. upload.updateProgress(Math.toIntExact(numberObjects), 0); upload.getProgress().setMessage(message); @@ -61,6 +64,7 @@ public void finishUpload(ImportUpload upload, long numberObjects, String message } public void updateMappedData(ImportUpload upload, ImportPreviewResponse response, String message) { + log.debug(message); // Save our results to the db JSON config = new JSON(); String json = config.getGson().toJson(response); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorData.java index 41f432f72..1a4a806b9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorData.java @@ -26,27 +26,27 @@ public class ProcessorData { - static int getNumNewObjects(Map> objectsByName) { + public static int getNumNewObjects(Map> objectsByName) { long numNewObjects = objectsByName.values().stream() .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW) .count(); return Math.toIntExact(numNewObjects); } - static int getNumExistingObjects(Map> objectsByName) { + public static int getNumExistingObjects(Map> objectsByName) { long numExistingObjects = objectsByName.values().stream() .filter(preview -> preview != null && preview.getState() == ImportObjectState.EXISTING) .count(); return Math.toIntExact(numExistingObjects); } - static List getNewObjects(Map> objectsByName) { + public static List getNewObjects(Map> objectsByName) { return objectsByName.values().stream() .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW) .map(preview -> preview.getBrAPIObject()) .collect(Collectors.toList()); } - static Map getMutationsByObjectId(Map> objectsByName, Function dbIdFilter) { + public static Map getMutationsByObjectId(Map> objectsByName, Function dbIdFilter) { return objectsByName.entrySet().stream() .filter(entry -> ImportObjectState.MUTATED == entry.getValue().getState()) .collect(Collectors diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 9983b1aaa..8092a01e9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -16,6 +16,8 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang3.StringUtils; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; @@ -30,6 +32,10 @@ public class ExperimentUtilities { public static final String TIMESTAMP_PREFIX = "TS:"; public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; public static final String MIDNIGHT = "T00:00:00-00:00"; + public static final String MULTIPLE_EXP_TITLES = "File contains more than one Experiment Title"; + public static final String PREEXISTING_EXPERIMENT_TITLE = "Experiment Title already exists"; + + public static List importRowsToExperimentObservations(List importRows) { return importRows.stream() @@ -46,4 +52,11 @@ public static boolean validDateTimeValue(String value) { } return true; } + + public static String getObservationHash(String observationUnitName, String variableName, String studyName) { + String concat = DigestUtils.sha256Hex(observationUnitName) + + DigestUtils.sha256Hex(variableName) + + DigestUtils.sha256Hex(StringUtils.defaultString(studyName)); + return DigestUtils.sha256Hex(concat); + } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java index e05b1cf25..3fe0ba4ee 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/steps/PopulateModifiedPendingImportObjectsStep.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; +package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow.steps; // TODO: think this would be for other workflow only public class PopulateModifiedPendingImportObjectsStep { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java index 213411078..96c54712b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -21,6 +21,7 @@ import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.model.ProgramLocation; @@ -40,4 +41,6 @@ public class PendingData { private Map> locationByName; private Map> obsVarDatasetByName; private Map> existingGermplasmByGID; + // TODO: see if we can change this to match PendingImport<> + private Map existingObsByObsHash; } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 4666ac0f1..e59226b81 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -18,16 +18,32 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; import org.breedinginsight.brapps.importer.model.workflow.Workflow; -import org.breedinginsight.brapps.importer.services.pipeline.Pipeline; +import org.breedinginsight.brapps.importer.services.ImportStatusService; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.CommitPendingImportObjectsStep; import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.PopulateExistingPendingImportObjectsStep; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.ProcessStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.PopulateNewPendingImportObjectsStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentPhenotypeService; import javax.inject.Inject; import javax.inject.Named; -import javax.inject.Provider; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; /** * This class represents a workflow for creating a new experiment. The bean name must match the appropriate bean column @@ -35,28 +51,65 @@ */ @Prototype +@Slf4j @Named("CreateNewExperimentWorkflow") public class CreateNewExperimentWorkflow implements Workflow { - private final Provider getExistingStepProvider; - private final Provider processStepProvider; + private final PopulateExistingPendingImportObjectsStep populateExistingPendingImportObjectsStep; + private final PopulateNewPendingImportObjectsStep populateNewPendingImportObjectsStep; + private final CommitPendingImportObjectsStep commitPendingImportObjectsStep; + private final ImportStatusService statusService; + private final ExperimentPhenotypeService experimentPhenotypeService; @Inject - public CreateNewExperimentWorkflow(Provider getExistingStepProvider, - Provider processStepProvider) { - this.getExistingStepProvider = getExistingStepProvider; - this.processStepProvider = processStepProvider; + public CreateNewExperimentWorkflow(PopulateExistingPendingImportObjectsStep populateExistingPendingImportObjectsStep, + PopulateNewPendingImportObjectsStep populateNewPendingImportObjectsStep, + CommitPendingImportObjectsStep commitPendingImportObjectsStep, + ImportStatusService statusService, + ExperimentPhenotypeService experimentPhenotypeService) { + this.populateExistingPendingImportObjectsStep = populateExistingPendingImportObjectsStep; + this.populateNewPendingImportObjectsStep = populateNewPendingImportObjectsStep; + this.commitPendingImportObjectsStep = commitPendingImportObjectsStep; + this.statusService = statusService; + this.experimentPhenotypeService = experimentPhenotypeService; } @Override - public ProcessedData process(ImportContext context) { - // TODO - Pipeline pipeline = new Pipeline<>(getExistingStepProvider.get()) - .addProcessingStep(processStepProvider.get()); - ProcessedData processed = pipeline.execute(context); - - // TODO: return actual data - return processed; + public ImportPreviewResponse process(ImportContext context) { + + ImportUpload upload = context.getUpload(); + boolean commit = context.isCommit(); + + // Make sure the file does not contain obs unit ids before proceeding + if (containsObsUnitIDs(context)) { + // TODO: get file name + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Error detected in file, XXX.xls. ObsUnitIDs are detected. Import cannot proceed"); + } + + statusService.updateMessage(upload, "Checking existing experiment objects in brapi service and mapping data"); + + ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(context); + ProcessContext processContext = populateExistingPendingImportObjectsStep.process(context, phenotypeData); + ProcessedData processedData = populateNewPendingImportObjectsStep.process(processContext, phenotypeData); + ImportPreviewResponse response = buildImportPreviewResponse(processedData, upload); + + statusService.updateMappedData(upload, response, "Finished mapping data to brapi objects"); + + // preview data + if (!commit) { + statusService.updateOk(upload); + return response; + } + + // commit data + long totalObjects = getNewObjectCount(processedData); + statusService.startUpload(upload, totalObjects, "Starting upload to brapi service"); + statusService.updateMessage(upload, "Creating new experiment objects in brapi service"); + + commitPendingImportObjectsStep.process(processContext, processedData); + + statusService.finishUpload(upload, totalObjects, "Completed upload to brapi service"); + return response; } /** @@ -69,5 +122,38 @@ public ProcessedData process(ImportContext context) { public String getName() { return "CreateNewExperimentWorkflow"; } + + // TODO: move to shared area + private ImportPreviewResponse buildImportPreviewResponse(ProcessedData processedData, + ImportUpload upload) { + Map statistics = processedData.getStatistics(); + Map mappedBrAPIImport = processedData.getMappedBrAPIImport(); + + ImportPreviewResponse response = new ImportPreviewResponse(); + response.setStatistics(statistics); + List mappedBrAPIImportList = new ArrayList<>(mappedBrAPIImport.values()); + response.setRows(mappedBrAPIImportList); + response.setDynamicColumnNames(upload.getDynamicColumnNamesList()); + return response; + } + + // TODO: move to shared area + private long getNewObjectCount(ProcessedData processedData) { + // get total number of new brapi objects to create + long totalObjects = 0; + for (ImportPreviewStatistics stats : processedData.getStatistics().values()) { + totalObjects += stats.getNewObjectCount(); + } + return totalObjects; + } + + private boolean containsObsUnitIDs(ImportContext importContext) { + List importRows = importContext.getImportRows(); + return importRows.stream() + .anyMatch(row -> { + ExperimentObservation expRow = (ExperimentObservation) row; + return StringUtils.isNotBlank(expRow.getObsUnitID()); + }); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java index ea839298b..836ae2731 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; +import io.micronaut.context.annotation.Property; import org.apache.commons.lang3.StringUtils; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; @@ -18,7 +19,13 @@ import java.util.UUID; import java.util.function.Supplier; -public class CreatePendingImportPopulator implements PendingImportObjectPopulator { +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.MULTIPLE_EXP_TITLES; +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.PREEXISTING_EXPERIMENT_TITLE; + +public class CreatePendingImportPopulator implements PendingImportObjectPopulator { + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; @Override public PendingImportObject populateTrial(ImportContext importContext, @@ -54,14 +61,20 @@ public PendingImportObject populateTrial(ImportContext importContext } BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); - // TODO: move + // NOTE: moved up a level //trialByNameNoScope.put(importRow.getExpTitle(), trialPio); } return trialPio; } + @Override + public PendingImportObject populateStudy(ImportContext importContext, + Supplier expSeqValue, + ExperimentObservation importRow, + Supplier envNextVal) { + return null; + } - } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index 02ab60e3e..44c194cbf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -16,10 +16,62 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.request.BrAPIListNewRequest; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; -import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; -public class CommitPendingImportDataStep implements ProcessingStep { - +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Slf4j +public class CommitPendingImportObjectsStep { + + public void process(ProcessContext processContext, ProcessedData processedData) { + + PendingData pendingData = processContext.getPendingData(); + + List newTrials = ProcessorData.getNewObjects(pendingData.getTrialByNameNoScope()); + + List newLocations = ProcessorData.getNewObjects(pendingData.getLocationByName()) + .stream() + .map(location -> ProgramLocationRequest.builder() + .name(location.getName()) + .build()) + .collect(Collectors.toList()); + + List newStudies = ProcessorData.getNewObjects(pendingData.getStudyByNameNoScope()); + + List newDatasetRequests = ProcessorData.getNewObjects(pendingData.getObsVarDatasetByName()).stream().map(details -> { + BrAPIListNewRequest request = new BrAPIListNewRequest(); + request.setListName(details.getListName()); + request.setListType(details.getListType()); + request.setExternalReferences(details.getExternalReferences()); + request.setAdditionalInfo(details.getAdditionalInfo()); + request.data(details.getData()); + return request; + }).collect(Collectors.toList()); + + Map datasetNewDataById = ProcessorData + .getMutationsByObjectId(pendingData.getObsVarDatasetByName(), BrAPIListSummary::getListDbId); + + List newObservationUnits = ProcessorData.getNewObjects(pendingData.getObservationUnitByNameNoScope()); + + // filter out observations with no 'value' so they will not be saved + List newObservations = ProcessorData.getNewObjects(this.observationByHash) + .stream() + .filter(obs -> !obs.getValue().isBlank()) + .collect(Collectors.toList()); + + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index 8ee355dc3..d3f83d45b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -17,7 +17,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; import io.micronaut.context.annotation.Property; -import io.micronaut.context.annotation.Prototype; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -29,6 +28,7 @@ import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.*; @@ -37,18 +37,20 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedStudyService; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedTrialService; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentStudyService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentTrialService; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.model.Trait; import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; +import javax.inject.Singleton; import java.util.*; import java.util.stream.Collectors; @@ -57,9 +59,9 @@ * steps rather than another layer of services. */ -@Prototype +@Singleton @Slf4j -public class PopulateExistingPendingImportObjectsStep implements ProcessingStep { +public class PopulateExistingPendingImportObjectsStep { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final BrAPITrialDAO brAPITrialDAO; @@ -67,8 +69,9 @@ public class PopulateExistingPendingImportObjectsStep implements ProcessingStep< private final ProgramLocationService locationService; private final BrAPIListDAO brAPIListDAO; private final BrAPIGermplasmDAO brAPIGermplasmDAO; - private final SharedStudyService sharedStudyService; - private final SharedTrialService sharedTrialService; + private final BrAPIObservationDAO brAPIObservationDAO; + private final ExperimentStudyService experimentStudyService; + private final ExperimentTrialService experimentTrialService; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @@ -80,32 +83,34 @@ public PopulateExistingPendingImportObjectsStep(BrAPIObservationUnitDAO brAPIObs ProgramLocationService locationService, BrAPIListDAO brAPIListDAO, BrAPIGermplasmDAO brAPIGermplasmDAO, - SharedStudyService sharedStudyService, - SharedTrialService sharedTrialService) { + BrAPIObservationDAO brAPIObservationDAO, + ExperimentStudyService experimentStudyService, + ExperimentTrialService experimentTrialService) { this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.brAPITrialDAO = brAPITrialDAO; this.brAPIStudyDAO = brAPIStudyDAO; this.locationService = locationService; this.brAPIListDAO = brAPIListDAO; this.brAPIGermplasmDAO = brAPIGermplasmDAO; - this.sharedStudyService = sharedStudyService; - this.sharedTrialService = sharedTrialService; + this.brAPIObservationDAO = brAPIObservationDAO; + this.experimentStudyService = experimentStudyService; + this.experimentTrialService = experimentTrialService; } - @Override - public ProcessContext process(ImportContext input) { + public ProcessContext process(ImportContext input, ProcessedPhenotypeData phenotypeData) { List experimentImportRows = ExperimentUtilities.importRowsToExperimentObservations(input.getImportRows()); Program program = input.getProgram(); // Populate pending objects with existing status Map> observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); - Map> trialByNameNoScope = sharedTrialService.initializeTrialByNameNoScope(program, observationUnitByNameNoScope, experimentImportRows); + Map> trialByNameNoScope = experimentTrialService.initializeTrialByNameNoScope(program, observationUnitByNameNoScope, experimentImportRows); Map> studyByNameNoScope = initializeStudyByNameNoScope(program, trialByNameNoScope, observationUnitByNameNoScope, experimentImportRows); // interesting we're using our data model instead of brapi for locations Map> locationByName = initializeUniqueLocationNames(program, studyByNameNoScope, experimentImportRows); Map> obsVarDatasetByName = initializeObsVarDatasetByName(program, trialByNameNoScope, experimentImportRows); Map> existingGermplasmByGID = initializeExistingGermplasmByGID(program, observationUnitByNameNoScope, experimentImportRows); + Map existingObsByObsHash = fetchExistingObservations(phenotypeData.getReferencedTraits(), studyByNameNoScope, program); PendingData existing = PendingData.builder() .observationUnitByNameNoScope(observationUnitByNameNoScope) @@ -114,6 +119,7 @@ public ProcessContext process(ImportContext input) { .locationByName(locationByName) .obsVarDatasetByName(obsVarDatasetByName) .existingGermplasmByGID(existingGermplasmByGID) + .existingObsByObsHash(existingObsByObsHash) .build(); return ProcessContext.builder() @@ -253,7 +259,7 @@ private Map> initializeStudyByNameNoScop UUID experimentId = trial.get().getId(); existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); for (BrAPIStudy existingStudy : existingStudies) { - sharedStudyService.processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); + experimentStudyService.processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); } } catch (ApiException e) { log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); @@ -302,9 +308,9 @@ private void initializeStudiesForExistingObservationUnits( .getStudyDbId()) .collect(Collectors.toSet()); - List studies = sharedStudyService.fetchStudiesByDbId(studyDbIds, program); + List studies = experimentStudyService.fetchStudiesByDbId(studyDbIds, program); for (BrAPIStudy study : studies) { - sharedStudyService.processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); + experimentStudyService.processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); } } @@ -487,4 +493,63 @@ private ArrayList getGermplasmByAccessionNumber( return resultGermplasm; } + /** + * Fetches existing observations based on the given referenced traits, studyByNameNoScope map, and program. + * + * @param referencedTraits The list of referenced traits. + * @param studyByNameNoScope The map of studies by name without scope. + * @param program The program. + * @return A map of existing observations with their unique keys. + */ + private Map fetchExistingObservations(List referencedTraits, + Map> studyByNameNoScope, + Program program) { + Set ouDbIds = new HashSet<>(); + Set variableDbIds = new HashSet<>(); + Map variableNameByDbId = new HashMap<>(); + Map ouNameByDbId = new HashMap<>(); + Map studyNameByDbId = studyByNameNoScope.values() + .stream() + .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) + .map(PendingImportObject::getBrAPIObject) + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); + + studyNameByDbId.keySet().forEach(studyDbId -> { + try { + brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { + if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { + ouDbIds.add(ou.getObservationUnitDbId()); + } + ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + }); + } catch (ApiException e) { + throw new RuntimeException(e); + } + }); + + for (Trait referencedTrait : referencedTraits) { + variableDbIds.add(referencedTrait.getObservationVariableDbId()); + variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); + } + + List existingObservations = new ArrayList<>(); + try { + existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); + } catch (ApiException e) { + throw new RuntimeException(e); + } + + return existingObservations.stream() + .map(obs -> { + String studyName = studyNameByDbId.get(obs.getStudyDbId()); + String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); + String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); + + String key = ExperimentUtilities.getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); + + return Map.entry(key, obs); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index 13870fc51..cd4faf164 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -1,4 +1,601 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingImportObjectData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.PendingImportObjectPopulator; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentSeasonService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentValidateService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.model.User; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; +import org.jooq.DSLContext; +import tech.tablesaw.api.Table; +import org.breedinginsight.model.Trait; +import tech.tablesaw.columns.Column; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.math.BigInteger; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.*; + +@Singleton +@Slf4j public class PopulateNewPendingImportObjectsStep { -} + + private final ExperimentValidateService experimentValidateService; + private final ExperimentSeasonService experimentSeasonService; + private final BrAPIObservationDAO brAPIObservationDAO; + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private final DSLContext dsl; + + @Inject + public PopulateNewPendingImportObjectsStep(ExperimentValidateService experimentValidateService, + ExperimentSeasonService experimentSeasonService, + BrAPIObservationDAO brAPIObservationDAO, + BrAPIObservationUnitDAO brAPIObservationUnitDAO, + DSLContext dsl) { + this.experimentValidateService = experimentValidateService; + this.experimentSeasonService = experimentSeasonService; + this.brAPIObservationDAO = brAPIObservationDAO; + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + this.dsl = dsl; + } + + public ProcessedData process(ProcessContext context, ProcessedPhenotypeData phenotypeData) { + + Table data = context.getImportContext().getData(); + ImportUpload upload = context.getImportContext().getUpload(); + ImportContext importContext = context.getImportContext(); + + + + // TODO: implement + return new ProcessedData(); + } + + + // initNew + // TODO: move to shared service + private void populatePendingImportObjects(ImportContext importContext, + ProcessedPhenotypeData phenotypeData, + PendingImportObjectPopulator pioPopulator) { + + List importRows = importContext.getImportRows(); + Program program = importContext.getProgram(); + boolean commit = importContext.isCommit(); + + Supplier expNextVal = getNextExperimentSequenceNumber(program); + Supplier envNextVal = getNextEnvironmentSequenceNumber(program); + + // NOTE: this was moved to the get existing step and kept in PendingData + // existingObsByObsHash = fetchExistingObservations(referencedTraits, program); + + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + + populateIndependentVariablePIOsForRow(); + + // ... (Common logic from the original method) + + PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); + + processObservations(importContext, phenotypeData, importRow, rowNum, commit, importRow.getEnvYear(), obsUnitPIO, studyPIO); + } + } + + + /** + * Returns a Supplier that generates the next experiment sequence number based on the given Program. + * + * @param program the Program for which to generate the next experiment sequence number + * @return a Supplier that generates the next experiment sequence number + * @throws HttpStatusException if the program is not properly configured for observation unit import + */ + private Supplier getNextExperimentSequenceNumber(Program program) { + String expSequenceName = program.getExpSequence(); + if (expSequenceName == null) { + log.error(String.format("Program, %s, is missing a value in the exp sequence column.", program.getName())); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for observation unit import"); + } + return () -> dsl.nextval(expSequenceName.toLowerCase()); + } + + /** + * Retrieves the next environment sequence number for a given program. + * + * @param program The program for which to get the next environment sequence number. + * @return A Supplier representing a function that generates the next environment sequence number. + * @throws HttpStatusException If the program is not properly configured for environment import. + */ + private Supplier getNextEnvironmentSequenceNumber(Program program) { + String envSequenceName = program.getEnvSequence(); + if (envSequenceName == null) { + log.error(String.format("Program, %s, is missing a value in the env sequence column.", program.getName())); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for environment import"); + } + return () -> dsl.nextval(envSequenceName.toLowerCase()); + } + + /** + * Populates independent variable PendingImportObjectData for a given row of import data. + * + * @param importContext The import context. + * @param phenotypeData The processed phenotype data. + * @param pendingData The pending data. + * @param importRow The import row. + * @param expNextVal The supplier for generating experiment next value. + * @param envNextVal The supplier for generating environment next value. + * @param pioPopulator The pending import object populator. + * @return The populated independent variable PendingImportObjectData. + * @throws MissingRequiredInfoException If any required information is missing. + * @throws UnprocessableEntityException If the entity is unprocessable. + * @throws ApiException If there is an API exception. + */ + private PendingImportObjectData populateIndependentVariablePIOsForRow(ImportContext importContext, + ProcessedPhenotypeData phenotypeData, + PendingData pendingData, + ExperimentObservation importRow, + Supplier expNextVal, + Supplier envNextVal, + PendingImportObjectPopulator pioPopulator) + throws MissingRequiredInfoException, UnprocessableEntityException, ApiException { + + Program program = importContext.getProgram(); + User user = importContext.getUser(); + boolean commit = importContext.isCommit(); + List referencedTraits = phenotypeData.getReferencedTraits(); + + PendingImportObject trialPIO = null; + try { + trialPIO = pioPopulator.populateTrial(importContext, pendingData, importRow, expNextVal); + + // moved up a level + if (trialPIO.getState() == ImportObjectState.NEW) { + pendingData.getTrialByNameNoScope().put(importRow.getExpTitle(), trialPIO); + } + } catch (UnprocessableEntityException e) { + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } + + String expSeqValue = null; + if (commit) { + expSeqValue = trialPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString(); + } + + if (commit) { + fetchOrCreateDatasetPIO(importRow, program, referencedTraits); + } + + fetchOrCreateLocationPIO(importRow); + + PendingImportObject studyPIO = fetchOrCreateStudyPIO(program, commit, expSeqValue, importRow, envNextVal); + + String envSeqValue = null; + if (commit) { + envSeqValue = studyPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER) + .getAsString(); + } + + PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); + + return PendingImportObjectData.builder() + .trialPIO(trialPIO) + .studyPIO(studyPIO) + .obsUnitPIO(obsUnitPIO) + .build(); + } + + private void processObservations(ImportContext importContext, ProcessedPhenotypeData phenotypeData, ExperimentObservation importRow, + int rowNum, boolean commit, String studyYear, + PendingImportObject obsUnitPIO, + PendingImportObject studyPIO) + throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { + Program program = importContext.getProgram(); + User user = importContext.getUser(); + List> phenotypeCols = phenotypeData.getPhenotypeCols(); + Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); + List referencedTraits = phenotypeData.getReferencedTraits(); + + for (Column column : phenotypeCols) { + // ... (Logic for processing observations) + } + } + + // TODO: move common code out + private void initNewBrapiData(ImportContext importContext, ProcessedPhenotypeData phenotypeData) + throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { + + Program program = importContext.getProgram(); + List importRows = importContext.getImportRows(); + User user = importContext.getUser(); + boolean commit = importContext.isCommit(); + Map existingObsByObsHash; + + List referencedTraits = phenotypeData.getReferencedTraits(); + List> phenotypeCols = phenotypeData.getPhenotypeCols(); + Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); + + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + + + for (Column column : phenotypeCols) { + //If associated timestamp column, add + String dateTimeValue = null; + if (timeStampColByPheno.containsKey(column.name())) { + dateTimeValue = timeStampColByPheno.get(column.name()).getString(rowNum); + //If no timestamp, set to midnight + if (!dateTimeValue.isBlank() && !validDateTimeValue(dateTimeValue)) { + dateTimeValue += MIDNIGHT; + } + } + + // get the study year either referenced from the observation unit or listed explicitly on the import row + // TODO: handle this different workflows + String studyYear = hasAllReferenceUnitIds ? studyPIO.getBrAPIObject().getSeasons().get(0) : importRow.getEnvYear(); + String seasonDbId = experimentSeasonService.yearToSeasonDbId(studyYear, program.getId()); + fetchOrCreateObservationPIO( + program, + user, + importRow, + column, //column.name() gets phenotype name + rowNum, + dateTimeValue, + commit, + seasonDbId, + obsUnitPIO, + studyPIO, + referencedTraits + ); + } + } + } + + private Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { + Set ouDbIds = new HashSet<>(); + Set variableDbIds = new HashSet<>(); + Map variableNameByDbId = new HashMap<>(); + Map ouNameByDbId = new HashMap<>(); + Map studyNameByDbId = studyByNameNoScope.values() + .stream() + .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) + .map(PendingImportObject::getBrAPIObject) + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); + + studyNameByDbId.keySet().forEach(studyDbId -> { + try { + brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { + if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { + ouDbIds.add(ou.getObservationUnitDbId()); + } + ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + }); + } catch (ApiException e) { + throw new RuntimeException(e); + } + }); + + for (Trait referencedTrait : referencedTraits) { + variableDbIds.add(referencedTrait.getObservationVariableDbId()); + variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); + } + + List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); + + return existingObservations.stream() + .map(obs -> { + String studyName = studyNameByDbId.get(obs.getStudyDbId()); + String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); + String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); + + String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); + + return Map.entry(key, obs); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private PendingImportObject fetchOrCreateTrialPIO( + Program program, + User user, + boolean commit, + ExperimentObservation importRow, + Supplier expNextVal + ) throws UnprocessableEntityException { + PendingImportObject trialPio; + + // use the prior trial if observation unit IDs are supplied + // TODO: handle multiple workflows + if (hasAllReferenceUnitIds) { + trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); + + // otherwise create a new trial, but there can be only one allowed + } else { + if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { + PendingImportObject envPio; + trialPio = trialByNameNoScope.get(importRow.getExpTitle()); + envPio = studyByNameNoScope.get(importRow.getEnv()); + + // creating new units for existing experiments and environments is not possible + if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && + (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ + throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); + } + } else if (!trialByNameNoScope.isEmpty()) { + throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); + } else { + UUID id = UUID.randomUUID(); + String expSeqValue = null; + if (commit) { + expSeqValue = expNextVal.get().toString(); + } + BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); + trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); + trialByNameNoScope.put(importRow.getExpTitle(), trialPio); + } + } + return trialPio; + } + + private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) throws UnprocessableEntityException { + PendingImportObject pio; + + // TODO: multiple workflows + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + String name = String.format("Observation Dataset [%s-%s]", + program.getKey(), + trialPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString()); + if (obsVarDatasetByName.containsKey(name)) { + pio = obsVarDatasetByName.get(name); + } else { + UUID id = UUID.randomUUID(); + BrAPIListDetails newDataset = importRow.constructDatasetDetails( + name, + id, + BRAPI_REFERENCE_SOURCE, + program, + trialPIO.getId().toString()); + pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); + trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); + if (ImportObjectState.EXISTING == trialPIO.getState()) { + trialPIO.setState(ImportObjectState.MUTATED); + } + obsVarDatasetByName.put(name, pio); + } + addObsVarsToDatasetDetails(pio, referencedTraits, program); + } + + private void fetchOrCreateLocationPIO(ExperimentObservation importRow) { + PendingImportObject pio; + // TODO: multiple workflows + String envLocationName = hasAllReferenceUnitIds ? + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName() : importRow.getEnvLocation(); + if (!locationByName.containsKey((importRow.getEnvLocation()))) { + ProgramLocation newLocation = new ProgramLocation(); + newLocation.setName(envLocationName); + pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); + this.locationByName.put(envLocationName, pio); + } + } + + private PendingImportObject fetchOrCreateStudyPIO( + Program program, + boolean commit, + String expSequenceValue, + ExperimentObservation importRow, + Supplier envNextVal + ) throws UnprocessableEntityException { + PendingImportObject pio; + // TODO: multiple workflows + if (hasAllReferenceUnitIds) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), + program.getKey() + ); + pio = studyByNameNoScope.get(studyName); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { + pio = studyByNameNoScope.get(importRow.getEnv()); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else { + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + UUID trialID = trialPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); + newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies + + // It is assumed that the study has only one season, And that the Years and not + // the dbId's are stored in getSeason() list. + String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season + if (commit) { + if(StringUtils.isNotBlank(year)) { + String seasonID = this.yearToSeasonDbId(year, program.getId()); + newStudy.setSeasons(Collections.singletonList(seasonID)); + } + } else { + addYearToStudyAdditionalInfo(program, newStudy, year); + } + + pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); + this.studyByNameNoScope.put(importRow.getEnv(), pio); + } + return pio; + } + + private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { + PendingImportObject pio; + String key = createObservationUnitKey(importRow); + // TODO: multiple workflows + if (hasAllReferenceUnitIds) { + pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); + } else if (observationUnitByNameNoScope.containsKey(key)) { + pio = observationUnitByNameNoScope.get(key); + } else { + String germplasmName = ""; + if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { + germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) + .getBrAPIObject() + .getGermplasmName(); + } + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; + UUID trialID = trialPIO.getId(); + UUID datasetId = null; + if (commit) { + datasetId = UUID.fromString(trialPIO.getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + } + PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); + + // check for existing units if this is an existing study + if (studyPIO.getBrAPIObject().getStudyDbId() != null) { + List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); + List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); + if (matchingOU.isEmpty()) { + throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); + } else { + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); + } + } else { + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); + } + this.observationUnitByNameNoScope.put(key, pio); + } + return pio; + } + + private void fetchOrCreateObservationPIO(Program program, + User user, + ExperimentObservation importRow, + Column column, + Integer rowNum, + String timeStampValue, + boolean commit, + String seasonDbId, + PendingImportObject obsUnitPIO, + PendingImportObject studyPIO, + List referencedTraits) throws ApiException, UnprocessableEntityException { + PendingImportObject pio; + BrAPIObservation newObservation; + String variableName = column.name(); + String value = column.getString(rowNum); + String key; + + // TODO: multiple workflows + if (hasAllReferenceUnitIds) { + String unitName = obsUnitPIO.getBrAPIObject().getObservationUnitName(); + String studyName = studyPIO.getBrAPIObject().getStudyName(); + key = getObservationHash(studyName + unitName, variableName, studyName); + } else { + key = getImportObservationHash(importRow, variableName); + } + + if (existingObsByObsHash.containsKey(key)) { + if (!isObservationMatched(key, value, column, rowNum)){ + + // prior observation with updated value + newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); + if (!isValueMatched(key, value)){ + newObservation.setValue(value); + } else if (!StringUtils.isBlank(timeStampValue) && !isTimestampMatched(key, timeStampValue)) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timeStampValue)); + newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + } + pio = new PendingImportObject<>(ImportObjectState.MUTATED, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); + } else { + + // prior observation + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(existingObsByObsHash.get(key), BrAPIObservation.class, program)); + } + + observationByHash.put(key, pio); + } else if (!this.observationByHash.containsKey(key)){ + + // new observation + // TODO: multiple workflows + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + + UUID trialID = trialPIO.getId(); + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject(), commit, program, user, BRAPI_REFERENCE_SOURCE, trialID, studyID, obsUnitPIO.getId(), id); + //NOTE: Can't parse invalid timestamp value, so have to skip if invalid. + // Validation error should be thrown for offending value, but that doesn't happen until later downstream + if (timeStampValue != null && !timeStampValue.isBlank() && (validDateValue(timeStampValue) || validDateTimeValue(timeStampValue))) { + newObservation.setObservationTimeStamp(OffsetDateTime.parse(timeStampValue)); + } + + newObservation.setStudyDbId(studyPIO.getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations + + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); + this.observationByHash.put(key, pio); + } + } + + +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java index b6f6e7796..cc81a8498 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java @@ -20,18 +20,14 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -43,64 +39,55 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; import org.breedinginsight.brapps.importer.services.processors.experiment.services.PendingImportObjectPopulator; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedPhenotypeService; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedSeasonService; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.SharedValidateService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentPhenotypeService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentSeasonService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentValidateService; import org.breedinginsight.model.Program; -import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.model.User; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; import tech.tablesaw.api.Table; import org.breedinginsight.model.Trait; -import tech.tablesaw.columns.Column; import javax.inject.Inject; import java.math.BigInteger; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; import java.util.*; import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.*; @Prototype @Slf4j -public class ProcessStep implements ProcessingStep { +public class ProcessStep implements { - private final SharedValidateService sharedValidateService; - private final SharedSeasonService sharedSeasonService; - private final SharedPhenotypeService sharedPhenotypeService; + private final ExperimentValidateService experimentValidateService; + private final ExperimentSeasonService experimentSeasonService; + private final ExperimentPhenotypeService experimentPhenotypeService; private final BrAPIObservationDAO brAPIObservationDAO; private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final DSLContext dsl; @Inject - public ProcessStep(SharedValidateService sharedValidateService, - SharedSeasonService sharedSeasonService, - SharedPhenotypeService sharedPhenotypeService, + public ProcessStep(ExperimentValidateService experimentValidateService, + ExperimentSeasonService experimentSeasonService, + ExperimentPhenotypeService experimentPhenotypeService, BrAPIObservationDAO brAPIObservationDAO, BrAPIObservationUnitDAO brAPIObservationUnitDAO, DSLContext dsl) { - this.sharedValidateService = sharedValidateService; - this.sharedSeasonService = sharedSeasonService; - this.sharedPhenotypeService = sharedPhenotypeService; + this.experimentValidateService = experimentValidateService; + this.experimentSeasonService = experimentSeasonService; + this.experimentPhenotypeService = experimentPhenotypeService; this.brAPIObservationDAO = brAPIObservationDAO; this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.dsl = dsl; } - @Override public ProcessedData process(ProcessContext context) { Table data = context.getImportContext().getData(); ImportUpload upload = context.getImportContext().getUpload(); ImportContext importContext = context.getImportContext(); - ProcessedPhenotypeData phenotypeData = sharedPhenotypeService.extractPhenotypes(importContext); + ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(importContext); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPendingImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPendingImportService.java index a5d091d69..9a1af1c37 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPendingImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPendingImportService.java @@ -16,7 +16,7 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment.services; -public class SharedPendingImportService { +public class ExperimentPendingImportService { } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPhenotypeService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPhenotypeService.java index 6b9bd7dc5..ce744234d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPhenotypeService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentPhenotypeService.java @@ -37,13 +37,13 @@ @Singleton @Slf4j -public class SharedPhenotypeService { +public class ExperimentPhenotypeService { - private final SharedValidateService sharedValidateService; + private final ExperimentValidateService experimentValidateService; @Inject - public SharedPhenotypeService(SharedValidateService sharedValidateService) { - this.sharedValidateService = sharedValidateService; + public ExperimentPhenotypeService(ExperimentValidateService experimentValidateService) { + this.experimentValidateService = experimentValidateService; } /** @@ -58,7 +58,7 @@ public ProcessedPhenotypeData extractPhenotypes(ImportContext importContext) { Program program = importContext.getProgram(); DynamicColumnParser.DynamicColumnParseResult result = DynamicColumnParser.parse(data, upload.getDynamicColumnNames()); - List traits = sharedValidateService.verifyTraits(program.getId(), result); + List traits = experimentValidateService.verifyTraits(program.getId(), result); Map> timeStampColByPheno = new HashMap<>(); //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java index d168a8a16..5fd67a023 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java @@ -32,7 +32,7 @@ @Singleton @Slf4j -public class SharedSeasonService { +public class ExperimentSeasonService { private final BrAPISeasonDAO brAPISeasonDAO; @@ -40,7 +40,7 @@ public class SharedSeasonService { private Map yearToSeasonDbIdCache = new HashMap<>(); @Inject - public SharedSeasonService(BrAPISeasonDAO brAPISeasonDAO) { + public ExperimentSeasonService(BrAPISeasonDAO brAPISeasonDAO) { this.brAPISeasonDAO = brAPISeasonDAO; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentStudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentStudyService.java index 0e9aef883..c1bd4cd2c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentStudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentStudyService.java @@ -40,7 +40,7 @@ @Singleton @Slf4j -public class SharedStudyService { +public class ExperimentStudyService { private final BrAPISeasonDAO brAPISeasonDAO; private final BrAPIStudyDAO brAPIStudyDAO; @@ -49,8 +49,8 @@ public class SharedStudyService { private String BRAPI_REFERENCE_SOURCE; @Inject - public SharedStudyService(BrAPISeasonDAO brAPISeasonDAO, - BrAPIStudyDAO brAPIStudyDAO) { + public ExperimentStudyService(BrAPISeasonDAO brAPISeasonDAO, + BrAPIStudyDAO brAPIStudyDAO) { this.brAPISeasonDAO = brAPISeasonDAO; this.brAPIStudyDAO = brAPIStudyDAO; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java index d3d5efd1f..9db927345 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java @@ -41,17 +41,17 @@ @Singleton @Slf4j -public class SharedTrialService { +public class ExperimentTrialService { private final BrAPITrialDAO brAPITrialDAO; - private final SharedStudyService studyService; + private final ExperimentStudyService studyService; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @Inject - public SharedTrialService(BrAPITrialDAO brAPITrialDAO, - SharedStudyService studyService) { + public ExperimentTrialService(BrAPITrialDAO brAPITrialDAO, + ExperimentStudyService studyService) { this.brAPITrialDAO = brAPITrialDAO; this.studyService = studyService; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java index f2edfc3f5..af2bcb29e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java @@ -42,13 +42,13 @@ @Singleton @Slf4j -public class SharedValidateService { +public class ExperimentValidateService { private final OntologyService ontologyService; private final FileMappingUtil fileMappingUtil; @Inject - public SharedValidateService(OntologyService ontologyService, FileMappingUtil fileMappingUtil) { + public ExperimentValidateService(OntologyService ontologyService, FileMappingUtil fileMappingUtil) { this.ontologyService = ontologyService; this.fileMappingUtil = fileMappingUtil; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java index d189a461d..9ca689447 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java @@ -17,6 +17,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.services; +import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -36,6 +37,9 @@ PendingImportObject populateTrial( Supplier expNextVal ) throws UnprocessableEntityException; - - + PendingImportObject populateStudy( + ImportContext importContext, + Supplier expSeqValue, + ExperimentObservation importRow, + Supplier envNextVal); } From bce2c7fbcce36ee87698257b521b9f87ffd2aa95 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:11:36 -0400 Subject: [PATCH 26/50] change Workflow field names --- .../model/workflow/ImportWorkflow.java | 4 ++-- .../ExperimentWorkflowNavigator.java | 18 +++++++++--------- .../AppendOverwritePhenotypesWorkflow.java | 4 ++-- .../workflow/CreateNewExperimentWorkflow.java | 4 ++-- .../workflow/CreateNewEnvironmentWorkflow.java | 4 ++-- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java index b26a886bd..30dbac06a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java @@ -25,7 +25,7 @@ @ToString @AllArgsConstructor public class ImportWorkflow { - private String urlFragment; - private String displayName; + private String id; + private String name; private int order; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index bcf12d543..b43b859f5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -42,22 +42,22 @@ public enum Workflow { APPEND_OVERWRITE("append-dataset", "Append experimental dataset"), APPEND_ENVIRONMENT("append-environment", "Create new experimental environment"); - private String urlFragment; - private String displayName; + private String id; + private String name; - Workflow(String urlFragment, String displayName) { + Workflow(String id, String name) { - this.urlFragment = urlFragment; - this.displayName = displayName; + this.id = id; + this.name = name; } - public String getUrlFragment() { - return urlFragment; + public String getId() { + return id; } - public String getDisplayName() { return displayName; } + public String getName() { return name; } public boolean isEqual(String value) { - return Optional.ofNullable(urlFragment.equals(value)).orElse(false); + return Optional.ofNullable(id.equals(value)).orElse(false); } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index e64a2f88a..6ff074059 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -23,8 +23,8 @@ public AppendOverwritePhenotypesWorkflow(){ public Optional process(ImportServiceContext context) { // Workflow processing the context ImportWorkflow workflow = ImportWorkflow.builder() - .urlFragment(getWorkflow().getUrlFragment()) - .displayName(getWorkflow().getDisplayName()) + .id(getWorkflow().getId()) + .name(getWorkflow().getName()) .build(); // No-preview result diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 022727c5a..503e4e5f0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -23,8 +23,8 @@ public CreateNewExperimentWorkflow(){ public Optional process(ImportServiceContext context) { // Workflow processing the context ImportWorkflow workflow = ImportWorkflow.builder() - .urlFragment(getWorkflow().getUrlFragment()) - .displayName(getWorkflow().getDisplayName()) + .id(getWorkflow().getId()) + .name(getWorkflow().getName()) .build(); // No-preview result diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java index a5695e7f8..1b7eba9ce 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java @@ -23,8 +23,8 @@ public CreateNewEnvironmentWorkflow(){ public Optional process(ImportServiceContext context) { // Workflow processing the context ImportWorkflow workflow = ImportWorkflow.builder() - .urlFragment(getWorkflow().getUrlFragment()) - .displayName(getWorkflow().getDisplayName()) + .id(getWorkflow().getId()) + .name(getWorkflow().getName()) .build(); // No-preview result From 6060fcc990177a9c7e779af2cc66a07937cf33e4 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:29:14 -0400 Subject: [PATCH 27/50] use processorManager until Workflow business logic in place --- .../importer/model/imports/DomainImportService.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 5f89de64a..d826033fb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -21,6 +21,7 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; @@ -35,6 +36,7 @@ @Slf4j public abstract class DomainImportService implements BrAPIImportService { + // TODO: delete processor fields once WorkflowNavigator is used private final Provider experimentProcessorProvider; private final Provider processorManagerProvider; private final Workflow workflowNavigator; @@ -66,7 +68,15 @@ public ImportPreviewResponse process(ImportServiceContext context) log.info("Workflow: " + context.getWorkflow()); } - return workflowNavigator.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); + // TODO: return results from WorkflowNavigator once processing logic is in separate workflows + // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); + return processorManagerProvider.get().process(context.getBrAPIImports(), + List.of(experimentProcessorProvider.get()), + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); } } From 90f478becd85cd15db0d47a989ee399fb38d2256 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 10 Jun 2024 16:29:33 -0400 Subject: [PATCH 28/50] fix workflow navigator injection --- .../importer/model/imports/DomainImportService.java | 8 ++++---- .../experimentObservation/ExperimentImportService.java | 9 ++------- .../model/imports/germplasm/GermplasmImportService.java | 3 ++- .../imports/sample/SampleSubmissionImportService.java | 3 ++- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index d826033fb..13719e18d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -41,16 +41,16 @@ public abstract class DomainImportService implements BrAPIImportService { private final Provider processorManagerProvider; private final Workflow workflowNavigator; - @Inject + public DomainImportService(Provider experimentProcessorProvider, - Provider processorManagerProvider) + Provider processorManagerProvider, + Workflow workflowNavigator) { this.experimentProcessorProvider = experimentProcessorProvider; this.processorManagerProvider = processorManagerProvider; - this.workflowNavigator = getNavigator(); + this.workflowNavigator = workflowNavigator; } - protected abstract Workflow getNavigator(); @Override public String getMissingColumnMsg(String columnName) { return "Column heading does not match template or ontology"; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 60fe4bb3b..793506c73 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -38,19 +38,14 @@ public class ExperimentImportService extends DomainImportService { private final String IMPORT_TYPE_ID = "ExperimentImport"; - private final ExperimentWorkflowNavigator workflowNavigator; + // TODO: delete processor fields once WorkflowNavigator is used @Inject public ExperimentImportService(Provider experimentProcessorProvider, Provider processorManagerProvider, ExperimentWorkflowNavigator workflowNavigator) { - super(experimentProcessorProvider, processorManagerProvider); - this.workflowNavigator = workflowNavigator; - } - @Override - public Workflow getNavigator() { - return this.workflowNavigator; + super(experimentProcessorProvider, processorManagerProvider, workflowNavigator); } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index 3c3a65b4d..0caebe65e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -34,6 +34,7 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import java.util.ArrayList; import java.util.List; @Singleton @@ -60,7 +61,7 @@ public GermplasmImport getImportClass() { @Override public List getWorkflows() { - return null; + return new ArrayList<>(); } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java index 55848b0a1..eb7328ecf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java @@ -34,6 +34,7 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import java.util.ArrayList; import java.util.List; @Singleton @@ -62,7 +63,7 @@ public BrAPIImport getImportClass() { @Override public List getWorkflows() { - return null; + return new ArrayList<>(); } @Override From 62cf9ac438684991bb48358ca46eea1418a280eb Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:07:33 -0400 Subject: [PATCH 29/50] set the order field for workflows --- .../experiment/ExperimentWorkflowNavigator.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index b43b859f5..63bfcedbd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -29,12 +29,21 @@ public Optional process(ImportServiceContext context) { .findFirst(); } public List getWorkflows() { - return workflows.stream() + // Each workflow returns in the field workflow the metadata about the workflow that processed the import context. + // Loop over all workflows, processing a null context, to collect just the metadata + List workflowSummaryList = workflows.stream() .map(workflow->workflow.process(null)) .filter(Optional::isPresent) .map(Optional::get) .map(result->result.getWorkflow()) .collect(Collectors.toList()); + + // The order field for each workflow is set to the order in the list + for (int i = 0; i < workflowSummaryList.size(); i++) { + workflowSummaryList.get(i).setOrder(i); + } + + return workflowSummaryList; } public enum Workflow { From 57f5cfbfc10f937e1099fd5a017997a7102db78f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:43:08 -0400 Subject: [PATCH 30/50] fix npe --- .../brapps/importer/model/imports/DomainImportService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 13719e18d..af65af238 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -64,7 +64,7 @@ public List getWorkflows() { public ImportPreviewResponse process(ImportServiceContext context) throws Exception { - if (!context.getWorkflow().isEmpty()) { + if (context.getWorkflow() != null && !context.getWorkflow().isEmpty()) { log.info("Workflow: " + context.getWorkflow()); } From 644ca0c8e0b7a51080f7d647d90de54160ec481c Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 12 Jun 2024 14:54:25 -0400 Subject: [PATCH 31/50] Added commit step --- .../ExperimentImportService.java | 15 + .../importer/model/workflow/Workflow.java | 2 +- .../experiment/ExperimentUtilities.java | 215 +++++++ .../experiment/create/model/PendingData.java | 5 + .../workflow/CreateNewExperimentWorkflow.java | 16 +- .../CreatePendingImportPopulator.java | 80 --- .../steps/CommitPendingImportObjectsStep.java | 305 ++++++++- ...ulateExistingPendingImportObjectsStep.java | 8 +- .../PopulateNewPendingImportObjectsStep.java | 308 +++++---- .../create/workflow/steps/ProcessStep.java | 597 ------------------ .../ValidatePendingImportObjectsStep.java | 464 ++++++++++++++ .../services/ExperimentSeasonService.java | 5 +- .../services/ExperimentValidateService.java | 2 + .../PendingImportObjectPopulator.java | 45 -- 14 files changed, 1198 insertions(+), 869 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index c6f68b251..fededeea1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -23,6 +23,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; @@ -75,9 +76,23 @@ public ImportPreviewResponse process(ImportServiceContext context) if (context.getWorkflow() != null) { log.info("Workflow: " + context.getWorkflow().getName()); + + // TODO: change when workflows selection is ready + if (context.getWorkflow().getName().equals("CreateNewExperimentWorkflow")) { + ImportContext importContext = ImportContext.builder() + .importRows(context.getBrAPIImports()) + .user(context.getUser()) + .data(context.getData()) + .commit(context.isCommit()) + .upload(context.getUpload()) + .build(); + + return context.getWorkflow().process(importContext); + } } // TODO: change to calling workflow process instead of processor manager + // other workflows besides create will pass through to old flow response = processorManagerProvider.get().process(context.getBrAPIImports(), processors, context.getData(), diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java index 8b2165933..5d8cbe66e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -27,7 +27,7 @@ public interface Workflow { * @param context the import context containing the necessary data for processing * @return the processed data */ - ImportPreviewResponse process(ImportContext context); + ImportPreviewResponse process(ImportContext context) throws Exception; /** * Retrieves the name of the Workflow for logging display purposes. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 8092a01e9..f3b57fb03 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -16,16 +16,36 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment; +import com.google.gson.JsonObject; +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Scale; +import org.breedinginsight.model.Trait; +import tech.tablesaw.columns.Column; +import java.math.BigDecimal; +import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.*; import java.util.stream.Collectors; +@Slf4j public class ExperimentUtilities { public static final CharSequence COMMA_DELIMITER = ","; @@ -34,6 +54,7 @@ public class ExperimentUtilities { public static final String MIDNIGHT = "T00:00:00-00:00"; public static final String MULTIPLE_EXP_TITLES = "File contains more than one Experiment Title"; public static final String PREEXISTING_EXPERIMENT_TITLE = "Experiment Title already exists"; + public static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs"; @@ -59,4 +80,198 @@ public static String getObservationHash(String observationUnitName, String varia DigestUtils.sha256Hex(StringUtils.defaultString(studyName)); return DigestUtils.sha256Hex(concat); } + + /* + * this finds the YEAR from the season list on the BrAPIStudy and then + * will add the year to the additionalInfo-field of the BrAPIStudy + * */ + public static void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study) { + JsonObject additionalInfo = study.getAdditionalInfo(); + + //if it is already there, don't add it. + if(additionalInfo==null || additionalInfo.get(BrAPIAdditionalInfoFields.ENV_YEAR)==null) { + String year = study.getSeasons().get(0); + addYearToStudyAdditionalInfo(program, study, year); + } + } + + /* + * this will add the given year to the additionalInfo field of the BrAPIStudy (if it does not already exist) + * */ + public static void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study, String year) { + JsonObject additionalInfo = study.getAdditionalInfo(); + if (additionalInfo==null){ + additionalInfo = new JsonObject(); + study.setAdditionalInfo(additionalInfo); + } + if( additionalInfo.get(BrAPIAdditionalInfoFields.ENV_YEAR)==null) { + additionalInfo.addProperty(BrAPIAdditionalInfoFields.ENV_YEAR, year); + } + } + + public static String createObservationUnitKey(ExperimentObservation importRow) { + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); + } + + public static String createObservationUnitKey(String studyName, String obsUnitName) { + return studyName + obsUnitName; + } + + public static String getImportObservationHash(ExperimentObservation importRow, String variableName) { + return getObservationHash(createObservationUnitKey(importRow), variableName, importRow.getEnv()); + } + + public static String getVariableNameFromColumn(Column column) { + // TODO: timestamp stripping? + return column.name(); + } + + // TODO: common validation stuff, could probably be moved somewhere more specific to validation + public static void addRowError(String field, String errorMessage, ValidationErrors validationErrors, int rowNum) { + ValidationError ve = new ValidationError(field, errorMessage, HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(rowNum + 2, ve); // +2 instead of +1 to account for the column header row. + } + + // TODO: will have different pending data objects between workflows so not totally reusable as-is + // could probably just pass in actual underlying maps + public static boolean isObservationMatched(ProcessedPhenotypeData phenotypeData, + PendingData pendingData, + String observationHash, + String value, + Column phenoCol, + Integer rowNum) { + Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); + + if (timeStampColByPheno.isEmpty() || !timeStampColByPheno.containsKey(phenoCol.name())) { + return isValueMatched(pendingData, observationHash, value); + } else { + String importObsTimestamp = timeStampColByPheno.get(phenoCol.name()).getString(rowNum); + return isTimestampMatched(pendingData, observationHash, importObsTimestamp) && isValueMatched(pendingData, observationHash, value); + } + } + + // TODO: will have different pending data objects between workflows so not totally reusable as-is + // could probably just pass in actual underlying maps + public static boolean isValueMatched(PendingData pendingData, String observationHash, String value) { + Map existingObsByObsHash = pendingData.getExistingObsByObsHash(); + + if (!existingObsByObsHash.containsKey(observationHash) || existingObsByObsHash.get(observationHash).getValue() == null) { + return value == null; + } + return existingObsByObsHash.get(observationHash).getValue().equals(value); + } + + // TODO: will have different pending data objects between workflows so not totally reusable as-is + // could probably just pass in actual underlying maps + public static boolean isTimestampMatched(PendingData pendingData, String observationHash, String timeStamp) { + OffsetDateTime priorStamp = null; + Map existingObsByObsHash = pendingData.getExistingObsByObsHash(); + + if(existingObsByObsHash.get(observationHash)!=null){ + priorStamp = existingObsByObsHash.get(observationHash).getObservationTimeStamp(); + } + if (priorStamp == null) { + return timeStamp == null; + } + boolean isMatched = false; + try { + isMatched = priorStamp.isEqual(OffsetDateTime.parse(timeStamp)); + } catch(DateTimeParseException e){ + //if timestamp is invalid DateTime not equal to validated priorStamp + log.error(e.getMessage(), e); + } + return isMatched; + } + + public static void validateObservationValue(Trait variable, String value, + String columnHeader, ValidationErrors validationErrors, int row) { + if (StringUtils.isBlank(value)) { + log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s\n\trow: %d", variable.getObservationVariableName(), row)); + return; + } + + if (isNAObservation(value)) { + log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s\n\trow: %d", variable.getObservationVariableName(), row)); + return; + } + + switch (variable.getScale().getDataType()) { + case NUMERICAL: + Optional number = validNumericValue(value); + if (number.isEmpty()) { + addRowError(columnHeader, "Non-numeric text detected detected", validationErrors, row); + } else if (!validNumericRange(number.get(), variable.getScale())) { + addRowError(columnHeader, "Value outside of min/max range detected", validationErrors, row); + } + break; + case DATE: + if (!validDateValue(value)) { + addRowError(columnHeader, "Incorrect date format detected. Expected YYYY-MM-DD", validationErrors, row); + } + break; + case ORDINAL: + if (!validCategory(variable.getScale().getCategories(), value)) { + addRowError(columnHeader, "Undefined ordinal category detected", validationErrors, row); + } + break; + case NOMINAL: + if (!validCategory(variable.getScale().getCategories(), value)) { + addRowError(columnHeader, "Undefined nominal category detected", validationErrors, row); + } + break; + default: + break; + } + + } + + public static Optional validNumericValue(String value) { + BigDecimal number; + try { + number = new BigDecimal(value); + } catch (NumberFormatException e) { + return Optional.empty(); + } + return Optional.of(number); + } + + public static boolean validNumericRange(BigDecimal value, Scale validValues) { + // account for empty min or max in valid determination + return (validValues.getValidValueMin() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMin())) >= 0) && + (validValues.getValidValueMax() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMax())) <= 0); + } + + public static boolean validDateValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } + + public static boolean validCategory(List categories, String value) { + Set categoryValues = categories.stream() + .map(category -> category.getValue().toLowerCase()) + .collect(Collectors.toSet()); + return categoryValues.contains(value.toLowerCase()); + } + + public static boolean isNAObservation(String value){ + return value.equalsIgnoreCase("NA"); + } + + public static void validateTimeStampValue(String value, + String columnHeader, ValidationErrors validationErrors, int row) { + if (StringUtils.isBlank(value)) { + log.debug(String.format("skipping validation of observation timestamp because there is no value.\n\tvariable: %s\n\trow: %d", columnHeader, row)); + return; + } + if (!validDateValue(value) && !validDateTimeValue(value)) { + addRowError(columnHeader, "Incorrect datetime format detected. Expected YYYY-MM-DD or YYYY-MM-DDThh:mm:ss+hh:mm", validationErrors, row); + } + + } + } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java index 96c54712b..70ba1b62a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -26,6 +26,7 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.model.ProgramLocation; +import java.util.HashMap; import java.util.Map; @Getter @@ -35,6 +36,7 @@ @AllArgsConstructor @NoArgsConstructor public class PendingData { + //NOTE: populated in populate existing and new steps depending on import private Map> observationUnitByNameNoScope; private Map> trialByNameNoScope; private Map> studyByNameNoScope; @@ -43,4 +45,7 @@ public class PendingData { private Map> existingGermplasmByGID; // TODO: see if we can change this to match PendingImport<> private Map existingObsByObsHash; + + // NOTE: populated in populate new step + private Map> observationByHash; } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index e59226b81..e32a5c29c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -21,7 +21,9 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; +import lombok.val; import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; @@ -37,7 +39,9 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.CommitPendingImportObjectsStep; import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.PopulateExistingPendingImportObjectsStep; import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.PopulateNewPendingImportObjectsStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.ValidatePendingImportObjectsStep; import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentPhenotypeService; +import org.breedinginsight.services.exceptions.ValidatorException; import javax.inject.Inject; import javax.inject.Named; @@ -58,6 +62,7 @@ public class CreateNewExperimentWorkflow implements Workflow { private final PopulateExistingPendingImportObjectsStep populateExistingPendingImportObjectsStep; private final PopulateNewPendingImportObjectsStep populateNewPendingImportObjectsStep; private final CommitPendingImportObjectsStep commitPendingImportObjectsStep; + private final ValidatePendingImportObjectsStep validatePendingImportObjectsStep; private final ImportStatusService statusService; private final ExperimentPhenotypeService experimentPhenotypeService; @@ -65,17 +70,19 @@ public class CreateNewExperimentWorkflow implements Workflow { public CreateNewExperimentWorkflow(PopulateExistingPendingImportObjectsStep populateExistingPendingImportObjectsStep, PopulateNewPendingImportObjectsStep populateNewPendingImportObjectsStep, CommitPendingImportObjectsStep commitPendingImportObjectsStep, + ValidatePendingImportObjectsStep validatePendingImportObjectsStep, ImportStatusService statusService, ExperimentPhenotypeService experimentPhenotypeService) { this.populateExistingPendingImportObjectsStep = populateExistingPendingImportObjectsStep; this.populateNewPendingImportObjectsStep = populateNewPendingImportObjectsStep; this.commitPendingImportObjectsStep = commitPendingImportObjectsStep; + this.validatePendingImportObjectsStep = validatePendingImportObjectsStep; this.statusService = statusService; this.experimentPhenotypeService = experimentPhenotypeService; } @Override - public ImportPreviewResponse process(ImportContext context) { + public ImportPreviewResponse process(ImportContext context) throws Exception { ImportUpload upload = context.getUpload(); boolean commit = context.isCommit(); @@ -91,6 +98,13 @@ public ImportPreviewResponse process(ImportContext context) { ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(context); ProcessContext processContext = populateExistingPendingImportObjectsStep.process(context, phenotypeData); ProcessedData processedData = populateNewPendingImportObjectsStep.process(processContext, phenotypeData); + ValidationErrors validationErrors = validatePendingImportObjectsStep.process(context); + + // short circuit if there were validation errors + if (validationErrors.hasErrors()) { + throw new ValidatorException(validationErrors); + } + ImportPreviewResponse response = buildImportPreviewResponse(processedData, upload); statusService.updateMappedData(upload, response, "Finished mapping data to brapi objects"); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java deleted file mode 100644 index 836ae2731..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreatePendingImportPopulator.java +++ /dev/null @@ -1,80 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; - -import io.micronaut.context.annotation.Property; -import org.apache.commons.lang3.StringUtils; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.PendingImportObjectPopulator; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; - -import java.math.BigInteger; -import java.util.Map; -import java.util.UUID; -import java.util.function.Supplier; - -import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.MULTIPLE_EXP_TITLES; -import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.PREEXISTING_EXPERIMENT_TITLE; - -public class CreatePendingImportPopulator implements PendingImportObjectPopulator { - - @Property(name = "brapi.server.reference-source") - private String BRAPI_REFERENCE_SOURCE; - - @Override - public PendingImportObject populateTrial(ImportContext importContext, - PendingData pendingData, - ExperimentObservation importRow, - Supplier expNextVal) - throws UnprocessableEntityException { - - PendingImportObject trialPio; - Program program = importContext.getProgram(); - User user = importContext.getUser(); - boolean commit = importContext.isCommit(); - Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); - Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); - - if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { - PendingImportObject envPio; - trialPio = trialByNameNoScope.get(importRow.getExpTitle()); - envPio = studyByNameNoScope.get(importRow.getEnv()); - - // creating new units for existing experiments and environments is not possible - if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && - (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ - throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); - } - } else if (!trialByNameNoScope.isEmpty()) { - throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); - } else { - UUID id = UUID.randomUUID(); - String expSeqValue = null; - if (commit) { - expSeqValue = expNextVal.get().toString(); - } - BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); - trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); - // NOTE: moved up a level - //trialByNameNoScope.put(importRow.getExpTitle(), trialPio); - } - - return trialPio; - } - - @Override - public PendingImportObject populateStudy(ImportContext importContext, - Supplier expSeqValue, - ExperimentObservation importRow, - Supplier envNextVal) { - return null; - } - - -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index 44c194cbf..fa02c76ed 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -16,30 +16,92 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; +import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.ProgramLocationService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.Utilities; +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; +@Singleton @Slf4j public class CommitPendingImportObjectsStep { + private final BrAPIListDAO brAPIListDAO; + private final BrAPITrialDAO brapiTrialDAO; + private final BrAPIStudyDAO brAPIStudyDAO; + private final BrAPIObservationDAO brAPIObservationDAO; + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private final ProgramLocationService locationService; + private final OntologyService ontologyService; + + @Inject + public CommitPendingImportObjectsStep(BrAPIListDAO brAPIListDAO, + BrAPITrialDAO brapiTrialDAO, + BrAPIStudyDAO brAPIStudyDAO, + BrAPIObservationDAO brAPIObservationDAO, + BrAPIObservationUnitDAO brAPIObservationUnitDAO, + ProgramLocationService locationService, + OntologyService ontologyService) { + this.brAPIListDAO = brAPIListDAO; + this.brapiTrialDAO = brapiTrialDAO; + this.brAPIStudyDAO = brAPIStudyDAO; + this.brAPIObservationDAO = brAPIObservationDAO; + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + this.locationService = locationService; + this.ontologyService = ontologyService; + } + + // TODO: some common code between workflows here that could be broken out, removed append/update specific code public void process(ProcessContext processContext, ProcessedData processedData) { PendingData pendingData = processContext.getPendingData(); + ImportContext importContext = processContext.getImportContext(); + + ImportUpload upload = importContext.getUpload(); + Program program = importContext.getProgram(); + + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + Map> obsVarDatasetByName = pendingData.getObsVarDatasetByName(); + Map> locationByName = pendingData.getLocationByName(); + Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); + Map> observationByHash = pendingData.getObservationByHash(); List newTrials = ProcessorData.getNewObjects(pendingData.getTrialByNameNoScope()); @@ -68,10 +130,251 @@ public void process(ProcessContext processContext, ProcessedData processedData) List newObservationUnits = ProcessorData.getNewObjects(pendingData.getObservationUnitByNameNoScope()); // filter out observations with no 'value' so they will not be saved - List newObservations = ProcessorData.getNewObjects(this.observationByHash) + List newObservations = ProcessorData.getNewObjects(observationByHash) .stream() .filter(obs -> !obs.getValue().isBlank()) .collect(Collectors.toList()); + AuthenticatedUser actingUser = new AuthenticatedUser(upload.getUpdatedByUser().getName(), new ArrayList<>(), upload.getUpdatedByUser().getId(), new ArrayList<>()); + + try { + List createdDatasets = new ArrayList<>(brAPIListDAO.createBrAPILists(newDatasetRequests, program.getId(), upload)); + createdDatasets.forEach(summary -> obsVarDatasetByName.get(summary.getListName()).getBrAPIObject().setListDbId(summary.getListDbId())); + + List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newTrials, program.getId(), upload)); + // set the DbId to the for each newly created trial + for (BrAPITrial createdTrial : createdTrials) { + String createdTrialName = Utilities.removeProgramKey(createdTrial.getTrialName(), program.getKey()); + trialByNameNoScope.get(createdTrialName) + .getBrAPIObject() + .setTrialDbId(createdTrial.getTrialDbId()); + } + + List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); + // set the DbId to the for each newly created location + for (ProgramLocation createdLocation : createdLocations) { + String createdLocationName = createdLocation.getName(); + locationByName.get(createdLocationName) + .getBrAPIObject() + .setLocationDbId(createdLocation.getLocationDbId()); + } + + updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); + List createdStudies = brAPIStudyDAO.createBrAPIStudies(newStudies, program.getId(), upload); + + // set the DbId to the for each newly created study + for (BrAPIStudy createdStudy : createdStudies) { + String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData(createdStudy.getStudyName(), program.getKey()); + studyByNameNoScope.get(createdStudy_name_no_key) + .getBrAPIObject() + .setStudyDbId(createdStudy.getStudyDbId()); + } + + updateObsUnitDependencyValues(pendingData, program.getKey()); + List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); + + // set the DbId to the for each newly created Observation Unit + for (BrAPIObservationUnit createdObservationUnit : createdObservationUnits) { + // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope + String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getStudyName(), program.getKey()); + String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getObservationUnitName(), program.getKey()); + String createdObsUnit_key = ExperimentUtilities.createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); + observationUnitByNameNoScope.get(createdObsUnit_key) + .getBrAPIObject() + .setObservationUnitDbId(createdObservationUnit.getObservationUnitDbId()); + } + + updateObservationDependencyValues(pendingData, program); + brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); + } catch (ApiException e) { + log.error("Error saving experiment import: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error saving experiment import", e); + throw new InternalServerException(e.getMessage(), e); + } + + // NOTE: removed mutated trials code + + datasetNewDataById.forEach((id, dataset) -> { + try { + List existingObsVarIds = brAPIListDAO.getListById(id, program.getId()).getResult().getData(); + List newObsVarIds = dataset + .getData() + .stream() + .filter(obsVarId -> !existingObsVarIds.contains(obsVarId)).collect(Collectors.toList()); + List obsVarIds = new ArrayList<>(existingObsVarIds); + obsVarIds.addAll(newObsVarIds); + dataset.setData(obsVarIds); + brAPIListDAO.updateBrAPIList(id, dataset, program.getId()); + } catch (ApiException e) { + log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating dataset observation variables: ", e); + throw new InternalServerException(e.getMessage(), e); + } + }); + + // NOTE: removed mutated observations code + + } + + private void updateStudyDependencyValues(PendingData pendingData, Map mappedBrAPIImport, String programKey) { + // update location DbIds in studies for all distinct locations + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + + mappedBrAPIImport.values() + .stream() + .map(PendingImport::getLocation) + .forEach(location -> updateStudyLocationDbId(pendingData, location)); + + // update trial DbIds in studies for all distinct trials + trialByNameNoScope.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(trial -> updateTrialDbId(pendingData, trial, programKey)); } + + private void updateTrialDbId(PendingData pendingData, BrAPITrial trial, String programKey) { + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + + studyByNameNoScope.values() + .stream() + .filter(study -> study.getBrAPIObject() + .getTrialName() + .equals(Utilities.removeProgramKey(trial.getTrialName(), programKey))) + .forEach(study -> study.getBrAPIObject() + .setTrialDbId(trial.getTrialDbId())); + } + + private void updateStudyLocationDbId(PendingData pendingData, PendingImportObject location) { + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + + studyByNameNoScope.values() + .stream() + .filter(study -> location.getId().toString() + .equals(study.getBrAPIObject() + .getLocationDbId())) + .forEach(study -> study.getBrAPIObject() + .setLocationDbId(location.getBrAPIObject().getLocationDbId())); + } + + private void updateObsUnitDependencyValues(PendingData pendingData, String programKey) { + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + Map> existingGermplasmByGID = pendingData.getExistingGermplasmByGID(); + + // update study DbIds + studyByNameNoScope.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(study -> updateStudyDbId(pendingData, study, programKey)); + + // update germplasm DbIds + existingGermplasmByGID.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(germplasm -> updateGermplasmDbId(pendingData, germplasm)); + } + + private void updateGermplasmDbId(PendingData pendingData, BrAPIGermplasm germplasm) { + Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); + + observationUnitByNameNoScope.values() + .stream() + .filter(obsUnit -> germplasm.getAccessionNumber() != null && + germplasm.getAccessionNumber().equals(obsUnit + .getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.GID).getAsString())) + .forEach(obsUnit -> obsUnit.getBrAPIObject() + .setGermplasmDbId(germplasm.getGermplasmDbId())); + } + + private void updateStudyDbId(PendingData pendingData, BrAPIStudy study, String programKey) { + Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); + + observationUnitByNameNoScope.values() + .stream() + .filter(obsUnit -> obsUnit.getBrAPIObject() + .getStudyName() + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), programKey))) + .forEach(obsUnit -> { + obsUnit.getBrAPIObject() + .setStudyDbId(study.getStudyDbId()); + obsUnit.getBrAPIObject() + .setTrialDbId(study.getTrialDbId()); + }); + } + + + private void updateObservationDependencyValues(PendingData pendingData, Program program) { + String programKey = program.getKey(); + Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); + Map> observationByHash = pendingData.getObservationByHash(); + + // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds + observationUnitByNameNoScope.values().stream() + .map(PendingImportObject::getBrAPIObject) + .forEach(obsUnit -> updateObservationDbIds(pendingData, obsUnit, programKey)); + + // Update ObservationVariable DbIds + List traits = getTraitList(program); + CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); + for ( Trait trait: traits) { + traitMap.put(trait.getObservationVariableName(),trait); + } + for (PendingImportObject observation : observationByHash.values()) { + String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); + if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { + String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); + observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); + } + } + } + + // Update each ovservation's observationUnit DbId, study DbId, and germplasm DbId + private void updateObservationDbIds(PendingData pendingData, BrAPIObservationUnit obsUnit, String programKey) { + Map> observationByHash = pendingData.getObservationByHash(); + + // FILTER LOGIC: Match on Env and Exp Unit ID + observationByHash.values() + .stream() + .filter(obs -> obs.getBrAPIObject() + .getAdditionalInfo() != null + && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null + && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) + .getAsString() + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getStudyName(), programKey)) + && Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getBrAPIObject().getObservationUnitName(), programKey) + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getObservationUnitName(), programKey)) + ) + .forEach(obs -> { + if (StringUtils.isBlank(obs.getBrAPIObject().getObservationUnitDbId())) { + obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); + } + obs.getBrAPIObject().setStudyDbId(obsUnit.getStudyDbId()); + obs.getBrAPIObject().setGermplasmDbId(obsUnit.getGermplasmDbId()); + }); + } + + private List getTraitList(Program program) { + try { + return ontologyService.getTraitsByProgramId(program.getId(), true); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + throw new InternalServerException(e.toString(), e); + } + } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index d3f83d45b..22799a025 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -207,19 +207,13 @@ private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservatio ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); - observationUnitByName.put(createObservationUnitKey(row), + observationUnitByName.put(ExperimentUtilities.createObservationUnitKey(row), new PendingImportObject<>(ImportObjectState.EXISTING, brAPIObservationUnit, UUID.fromString(idRef.getReferenceId()))); } - private String createObservationUnitKey(ExperimentObservation importRow) { - return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); - } - private String createObservationUnitKey(String studyName, String obsUnitName) { - return studyName + obsUnitName; - } /** * Initializes studies by name without scope. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index cd4faf164..2d22895fc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -16,14 +16,17 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; +import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpStatus; import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.xmlbeans.impl.xb.xsdschema.ImportDocument; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; @@ -36,11 +39,11 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingImportObjectData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.PendingImportObjectPopulator; import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentSeasonService; import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentValidateService; import org.breedinginsight.model.Program; @@ -75,6 +78,9 @@ public class PopulateNewPendingImportObjectsStep { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final DSLContext dsl; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + @Inject public PopulateNewPendingImportObjectsStep(ExperimentValidateService experimentValidateService, ExperimentSeasonService experimentSeasonService, @@ -88,12 +94,14 @@ public PopulateNewPendingImportObjectsStep(ExperimentValidateService experimentV this.dsl = dsl; } - public ProcessedData process(ProcessContext context, ProcessedPhenotypeData phenotypeData) { + public ProcessedData process(ProcessContext context, ProcessedPhenotypeData phenotypeData) + throws MissingRequiredInfoException, UnprocessableEntityException, ApiException { Table data = context.getImportContext().getData(); ImportUpload upload = context.getImportContext().getUpload(); ImportContext importContext = context.getImportContext(); + populatePendingImportObjects(context, phenotypeData); // TODO: implement @@ -101,15 +109,18 @@ public ProcessedData process(ProcessContext context, ProcessedPhenotypeData phen } - // initNew + // NOTE: was called initNew + // initNewBrapiData(importRows, phenotypeCols, program, user, referencedTraits, commit); // TODO: move to shared service - private void populatePendingImportObjects(ImportContext importContext, - ProcessedPhenotypeData phenotypeData, - PendingImportObjectPopulator pioPopulator) { + private void populatePendingImportObjects(ProcessContext processContext, + ProcessedPhenotypeData phenotypeData) + throws MissingRequiredInfoException, UnprocessableEntityException, ApiException { + ImportContext importContext = processContext.getImportContext(); List importRows = importContext.getImportRows(); Program program = importContext.getProgram(); boolean commit = importContext.isCommit(); + PendingData pendingData = processContext.getPendingData(); Supplier expNextVal = getNextExperimentSequenceNumber(program); Supplier envNextVal = getNextEnvironmentSequenceNumber(program); @@ -120,7 +131,7 @@ private void populatePendingImportObjects(ImportContext importContext, for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - populateIndependentVariablePIOsForRow(); + populateIndependentVariablePIOsForRow(importContext, phenotypeData, pendingData, importRow, expNextVal, envNextVal); // ... (Common logic from the original method) @@ -130,7 +141,7 @@ private void populatePendingImportObjects(ImportContext importContext, } } - + // TODO: these sequence methods could be moved to common area /** * Returns a Supplier that generates the next experiment sequence number based on the given Program. * @@ -172,19 +183,18 @@ private Supplier getNextEnvironmentSequenceNumber(Program program) { * @param importRow The import row. * @param expNextVal The supplier for generating experiment next value. * @param envNextVal The supplier for generating environment next value. - * @param pioPopulator The pending import object populator. * @return The populated independent variable PendingImportObjectData. * @throws MissingRequiredInfoException If any required information is missing. * @throws UnprocessableEntityException If the entity is unprocessable. * @throws ApiException If there is an API exception. */ + // TODO: this could potentially be made reusable between workflows in the future private PendingImportObjectData populateIndependentVariablePIOsForRow(ImportContext importContext, ProcessedPhenotypeData phenotypeData, PendingData pendingData, ExperimentObservation importRow, Supplier expNextVal, - Supplier envNextVal, - PendingImportObjectPopulator pioPopulator) + Supplier envNextVal) throws MissingRequiredInfoException, UnprocessableEntityException, ApiException { Program program = importContext.getProgram(); @@ -194,9 +204,9 @@ private PendingImportObjectData populateIndependentVariablePIOsForRow(ImportCont PendingImportObject trialPIO = null; try { - trialPIO = pioPopulator.populateTrial(importContext, pendingData, importRow, expNextVal); + trialPIO = populateTrial(importContext, pendingData, importRow, expNextVal); - // moved up a level + // NOTE: moved up a level if (trialPIO.getState() == ImportObjectState.NEW) { pendingData.getTrialByNameNoScope().put(importRow.getExpTitle(), trialPIO); } @@ -210,15 +220,15 @@ private PendingImportObjectData populateIndependentVariablePIOsForRow(ImportCont .getAdditionalInfo() .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) .getAsString(); - } - if (commit) { - fetchOrCreateDatasetPIO(importRow, program, referencedTraits); + // updates pendingData obsVarDatasetByName PIO + fetchOrCreateDatasetPIO(importContext, pendingData, importRow, referencedTraits); } - fetchOrCreateLocationPIO(importRow); + // updates pendingData locationByName PIO + fetchOrCreateLocationPIO(pendingData, importRow); - PendingImportObject studyPIO = fetchOrCreateStudyPIO(program, commit, expSeqValue, importRow, envNextVal); + PendingImportObject studyPIO = fetchOrCreateStudyPIO(importContext, pendingData, expSeqValue, importRow, envNextVal); String envSeqValue = null; if (commit) { @@ -228,7 +238,7 @@ private PendingImportObjectData populateIndependentVariablePIOsForRow(ImportCont .getAsString(); } - PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); + PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(importContext, pendingData, envSeqValue, importRow); return PendingImportObjectData.builder() .trialPIO(trialPIO) @@ -302,98 +312,67 @@ private void initNewBrapiData(ImportContext importContext, ProcessedPhenotypeDat } } - private Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { - Set ouDbIds = new HashSet<>(); - Set variableDbIds = new HashSet<>(); - Map variableNameByDbId = new HashMap<>(); - Map ouNameByDbId = new HashMap<>(); - Map studyNameByDbId = studyByNameNoScope.values() - .stream() - .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) - .map(PendingImportObject::getBrAPIObject) - .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); - - studyNameByDbId.keySet().forEach(studyDbId -> { - try { - brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { - if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { - ouDbIds.add(ou.getObservationUnitDbId()); - } - ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); - }); - } catch (ApiException e) { - throw new RuntimeException(e); - } - }); + public PendingImportObject populateTrial(ImportContext importContext, + PendingData pendingData, + ExperimentObservation importRow, + Supplier expNextVal) + throws UnprocessableEntityException { - for (Trait referencedTrait : referencedTraits) { - variableDbIds.add(referencedTrait.getObservationVariableDbId()); - variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); - } - - List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); - - return existingObservations.stream() - .map(obs -> { - String studyName = studyNameByDbId.get(obs.getStudyDbId()); - String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); - String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); - - String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); - - return Map.entry(key, obs); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - private PendingImportObject fetchOrCreateTrialPIO( - Program program, - User user, - boolean commit, - ExperimentObservation importRow, - Supplier expNextVal - ) throws UnprocessableEntityException { PendingImportObject trialPio; - - // use the prior trial if observation unit IDs are supplied - // TODO: handle multiple workflows - if (hasAllReferenceUnitIds) { - trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); - - // otherwise create a new trial, but there can be only one allowed + Program program = importContext.getProgram(); + User user = importContext.getUser(); + boolean commit = importContext.isCommit(); + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + + if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { + PendingImportObject envPio; + trialPio = trialByNameNoScope.get(importRow.getExpTitle()); + envPio = studyByNameNoScope.get(importRow.getEnv()); + + // creating new units for existing experiments and environments is not possible + if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && + (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ + throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); + } + } else if (!trialByNameNoScope.isEmpty()) { + throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); } else { - if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { - PendingImportObject envPio; - trialPio = trialByNameNoScope.get(importRow.getExpTitle()); - envPio = studyByNameNoScope.get(importRow.getEnv()); - - // creating new units for existing experiments and environments is not possible - if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && - (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ - throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); - } - } else if (!trialByNameNoScope.isEmpty()) { - throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); - } else { - UUID id = UUID.randomUUID(); - String expSeqValue = null; - if (commit) { - expSeqValue = expNextVal.get().toString(); - } - BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); - trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); - trialByNameNoScope.put(importRow.getExpTitle(), trialPio); + UUID id = UUID.randomUUID(); + String expSeqValue = null; + if (commit) { + expSeqValue = expNextVal.get().toString(); } + BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); + trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); + // NOTE: moved up a level + //trialByNameNoScope.put(importRow.getExpTitle(), trialPio); } + return trialPio; } - private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) throws UnprocessableEntityException { + /** + * Fetches or creates a dataset for import. + * + * @param importContext The import context + * @param pendingData The pending data containing information about the import (modified in place) + * @param importRow The import row representing an observation + * @param referencedTraits The list of referenced Trait objects + * @throws UnprocessableEntityException if the import data is invalid + */ + public void fetchOrCreateDatasetPIO(ImportContext importContext, + PendingData pendingData, + ExperimentObservation importRow, + List referencedTraits) throws UnprocessableEntityException { PendingImportObject pio; + Program program = importContext.getProgram(); + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + Map> obsVarDatasetByName = pendingData.getObsVarDatasetByName(); - // TODO: multiple workflows - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); + + // TODO: this is common to both workflows String name = String.format("Observation Dataset [%s-%s]", program.getKey(), trialPIO.getBrAPIObject() @@ -420,80 +399,137 @@ private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program pr addObsVarsToDatasetDetails(pio, referencedTraits, program); } - private void fetchOrCreateLocationPIO(ExperimentObservation importRow) { + /** + * Add observation variable IDs to the dataset details of a pending import object. + * + * @param pio The pending import object with BrAPIListDetails (modified in place) + * @param referencedTraits The list of referenced Trait objects + * @param program The Program object + */ + // TODO: common to both workflows + private void addObsVarsToDatasetDetails(PendingImportObject pio, List referencedTraits, Program program) { + BrAPIListDetails details = pio.getBrAPIObject(); + referencedTraits.forEach(trait -> { + String id = Utilities.appendProgramKey(trait.getObservationVariableName(), program.getKey()); + + // TODO - Don't append the key if connected to a brapi service operating with legacy data(no appended program key) + + if (!details.getData().contains(id) && ImportObjectState.EXISTING != pio.getState()) { + details.getData().add(id); + } + if (!details.getData().contains(id) && ImportObjectState.EXISTING == pio.getState()) { + details.getData().add(id); + pio.setState(ImportObjectState.MUTATED); + } + }); + } + + /** + * Fetches or creates a ProgramLocation object for import. + * + * @param pendingData The PendingData object containing information about the import (modified in place) + * @param importRow The ExperimentObservation object representing an observation + */ + public void fetchOrCreateLocationPIO(PendingData pendingData, ExperimentObservation importRow) { PendingImportObject pio; - // TODO: multiple workflows - String envLocationName = hasAllReferenceUnitIds ? - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName() : importRow.getEnvLocation(); + String envLocationName = importRow.getEnvLocation(); + // NOTE: other worklow referenced map specific to itself + Map> locationByName = pendingData.getLocationByName(); + + // TODO: common to both workflows if (!locationByName.containsKey((importRow.getEnvLocation()))) { ProgramLocation newLocation = new ProgramLocation(); newLocation.setName(envLocationName); pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); - this.locationByName.put(envLocationName, pio); + locationByName.put(envLocationName, pio); } } + /** + * Fetches an existing study or creates a new study for the given import row. + * + * @param importContext the import context + * @param pendingData the pending data + * @param expSequenceValue the experiment sequence value + * @param importRow the import row + * @param envNextVal the supplier for generating the next environment ID + * + * @return the pending import object containing the study + * + * @throws UnprocessableEntityException if the study is not processable + */ private PendingImportObject fetchOrCreateStudyPIO( - Program program, - boolean commit, + ImportContext importContext, + PendingData pendingData, String expSequenceValue, ExperimentObservation importRow, Supplier envNextVal ) throws UnprocessableEntityException { + + Program program = importContext.getProgram(); + boolean commit = importContext.isCommit(); + + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + Map> locationByName = pendingData.getLocationByName(); + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + PendingImportObject pio; // TODO: multiple workflows - if (hasAllReferenceUnitIds) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), - program.getKey() - ); - pio = studyByNameNoScope.get(studyName); - if (!commit){ - addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); - } - } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { + if (studyByNameNoScope.containsKey(importRow.getEnv())) { pio = studyByNameNoScope.get(importRow.getEnv()); if (!commit){ - addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + ExperimentUtilities.addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); } } else { - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + // NOTE: specific to this workflow, rest common + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); UUID id = UUID.randomUUID(); BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); - newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies + newStudy.setLocationDbId(locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies // It is assumed that the study has only one season, And that the Years and not // the dbId's are stored in getSeason() list. String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season if (commit) { if(StringUtils.isNotBlank(year)) { - String seasonID = this.yearToSeasonDbId(year, program.getId()); + // TODO: look at if this needs to be cleared across runs + String seasonID = experimentSeasonService.yearToSeasonDbId(year, program.getId()); newStudy.setSeasons(Collections.singletonList(seasonID)); } } else { - addYearToStudyAdditionalInfo(program, newStudy, year); + ExperimentUtilities.addYearToStudyAdditionalInfo(program, newStudy, year); } pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); - this.studyByNameNoScope.put(importRow.getEnv(), pio); + studyByNameNoScope.put(importRow.getEnv(), pio); } return pio; } - private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { + private PendingImportObject fetchOrCreateObsUnitPIO(ImportContext importContext, + PendingData pendingData, + String envSeqValue, + ExperimentObservation importRow) + throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { PendingImportObject pio; - String key = createObservationUnitKey(importRow); - // TODO: multiple workflows - if (hasAllReferenceUnitIds) { - pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); - } else if (observationUnitByNameNoScope.containsKey(key)) { + + Program program = importContext.getProgram(); + boolean commit = importContext.isCommit(); + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); + Map> existingGermplasmByGID = pendingData.getExistingGermplasmByGID(); + + String key = ExperimentUtilities.createObservationUnitKey(importRow); + // NOTE: removed other workflow + + if (observationUnitByNameNoScope.containsKey(key)) { pio = observationUnitByNameNoScope.get(key); } else { String germplasmName = ""; - if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { - germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) + if (existingGermplasmByGID.get(importRow.getGid()) != null) { + germplasmName = existingGermplasmByGID.get(importRow.getGid()) .getBrAPIObject() .getGermplasmName(); } @@ -505,7 +541,7 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra .getAdditionalInfo().getAsJsonObject() .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); } - PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); + PendingImportObject studyPIO = studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); @@ -515,14 +551,14 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); if (matchingOU.isEmpty()) { - throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); + throw new MissingRequiredInfoException(ExperimentUtilities.MISSING_OBS_UNIT_ID_ERROR); } else { pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); } } else { pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); } - this.observationUnitByNameNoScope.put(key, pio); + observationUnitByNameNoScope.put(key, pio); } return pio; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java deleted file mode 100644 index cc81a8498..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ProcessStep.java +++ /dev/null @@ -1,597 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; - -import io.micronaut.context.annotation.Prototype; -import io.micronaut.http.HttpStatus; -import io.micronaut.http.exceptions.HttpStatusException; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.services.pipeline.ProcessingStep; -import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingImportObjectData; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.PendingImportObjectPopulator; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentPhenotypeService; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentSeasonService; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentValidateService; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.jooq.DSLContext; -import tech.tablesaw.api.Table; -import org.breedinginsight.model.Trait; - -import javax.inject.Inject; -import java.math.BigInteger; -import java.util.*; -import java.util.function.Supplier; - -@Prototype -@Slf4j -public class ProcessStep implements { - - private final ExperimentValidateService experimentValidateService; - private final ExperimentSeasonService experimentSeasonService; - private final ExperimentPhenotypeService experimentPhenotypeService; - private final BrAPIObservationDAO brAPIObservationDAO; - private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; - private final DSLContext dsl; - - @Inject - public ProcessStep(ExperimentValidateService experimentValidateService, - ExperimentSeasonService experimentSeasonService, - ExperimentPhenotypeService experimentPhenotypeService, - BrAPIObservationDAO brAPIObservationDAO, - BrAPIObservationUnitDAO brAPIObservationUnitDAO, - DSLContext dsl) { - this.experimentValidateService = experimentValidateService; - this.experimentSeasonService = experimentSeasonService; - this.experimentPhenotypeService = experimentPhenotypeService; - this.brAPIObservationDAO = brAPIObservationDAO; - this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; - this.dsl = dsl; - } - - public ProcessedData process(ProcessContext context) { - - Table data = context.getImportContext().getData(); - ImportUpload upload = context.getImportContext().getUpload(); - ImportContext importContext = context.getImportContext(); - - ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(importContext); - - - - // TODO: implement - return new ProcessedData(); - } - - -/* - // initNew - private void populatePendingImportObjects(ImportContext importContext, - ProcessedPhenotypeData phenotypeData, - PendingImportObjectPopulator pioPopulator) { - - List importRows = importContext.getImportRows(); - Program program = importContext.getProgram(); - - Supplier expNextVal = getNextExperimentSequenceNumber(program); - Supplier envNextVal = getNextEnvironmentSequenceNumber(program); - - // TODO: handle this - // existingObsByObsHash = fetchExistingObservations(referencedTraits, program); - - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - - populateIndependentVariablePIOsForRow(); - - // ... (Common logic from the original method) - - PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); - - processObservations(importContext, phenotypeData, importRow, rowNum, commit, importRow.getEnvYear(), obsUnitPIO, studyPIO); - } - } -*/ - - /** - * Returns a Supplier that generates the next experiment sequence number based on the given Program. - * - * @param program the Program for which to generate the next experiment sequence number - * @return a Supplier that generates the next experiment sequence number - * @throws HttpStatusException if the program is not properly configured for observation unit import - */ - private Supplier getNextExperimentSequenceNumber(Program program) { - String expSequenceName = program.getExpSequence(); - if (expSequenceName == null) { - log.error(String.format("Program, %s, is missing a value in the exp sequence column.", program.getName())); - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for observation unit import"); - } - return () -> dsl.nextval(expSequenceName.toLowerCase()); - } - - /** - * Retrieves the next environment sequence number for a given program. - * - * @param program The program for which to get the next environment sequence number. - * @return A Supplier representing a function that generates the next environment sequence number. - * @throws HttpStatusException If the program is not properly configured for environment import. - */ - private Supplier getNextEnvironmentSequenceNumber(Program program) { - String envSequenceName = program.getEnvSequence(); - if (envSequenceName == null) { - log.error(String.format("Program, %s, is missing a value in the env sequence column.", program.getName())); - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for environment import"); - } - return () -> dsl.nextval(envSequenceName.toLowerCase()); - } - - /** - * Populates independent variable PendingImportObjectData for a given row of import data. - * - * @param importContext The import context. - * @param phenotypeData The processed phenotype data. - * @param pendingData The pending data. - * @param importRow The import row. - * @param expNextVal The supplier for generating experiment next value. - * @param envNextVal The supplier for generating environment next value. - * @param pioPopulator The pending import object populator. - * @return The populated independent variable PendingImportObjectData. - * @throws MissingRequiredInfoException If any required information is missing. - * @throws UnprocessableEntityException If the entity is unprocessable. - * @throws ApiException If there is an API exception. - */ - private PendingImportObjectData populateIndependentVariablePIOsForRow(ImportContext importContext, - ProcessedPhenotypeData phenotypeData, - PendingData pendingData, - ExperimentObservation importRow, - Supplier expNextVal, - Supplier envNextVal, - PendingImportObjectPopulator pioPopulator) - throws MissingRequiredInfoException, UnprocessableEntityException, ApiException { - - Program program = importContext.getProgram(); - User user = importContext.getUser(); - boolean commit = importContext.isCommit(); - List referencedTraits = phenotypeData.getReferencedTraits(); - - PendingImportObject trialPIO = null; - try { - trialPIO = pioPopulator.populateTrial(importContext, importRow, expNextVal); - - // moved up a level - if (trialPIO.getState() == ImportObjectState.NEW) { - pendingData.getTrialByNameNoScope().put(importRow.getExpTitle(), trialPIO); - } - } catch (UnprocessableEntityException e) { - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); - } - - String expSeqValue = null; - if (commit) { - expSeqValue = trialPIO.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) - .getAsString(); - } - - if (commit) { - fetchOrCreateDatasetPIO(importRow, program, referencedTraits); - } - - fetchOrCreateLocationPIO(importRow); - - PendingImportObject studyPIO = fetchOrCreateStudyPIO(program, commit, expSeqValue, importRow, envNextVal); - - String envSeqValue = null; - if (commit) { - envSeqValue = studyPIO.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER) - .getAsString(); - } - - PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); - - return PendingImportObjectData.builder() - .trialPIO(trialPIO) - .studyPIO(studyPIO) - .obsUnitPIO(obsUnitPIO) - .build(); - } - -/* - private void processObservations(ImportContext importContext, ProcessedPhenotypeData phenotypeData, ExperimentObservation importRow, - int rowNum, boolean commit, String studyYear, - PendingImportObject obsUnitPIO, - PendingImportObject studyPIO) - throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { - Program program = importContext.getProgram(); - User user = importContext.getUser(); - List> phenotypeCols = phenotypeData.getPhenotypeCols(); - Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); - List referencedTraits = phenotypeData.getReferencedTraits(); - - for (Column column : phenotypeCols) { - // ... (Logic for processing observations) - } - } - - // TODO: move common code out - private void initNewBrapiData(ImportContext importContext, ProcessedPhenotypeData phenotypeData) - throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { - - Program program = importContext.getProgram(); - List importRows = importContext.getImportRows(); - User user = importContext.getUser(); - boolean commit = importContext.isCommit(); - Map existingObsByObsHash; - - List referencedTraits = phenotypeData.getReferencedTraits(); - List> phenotypeCols = phenotypeData.getPhenotypeCols(); - Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); - - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - - - for (Column column : phenotypeCols) { - //If associated timestamp column, add - String dateTimeValue = null; - if (timeStampColByPheno.containsKey(column.name())) { - dateTimeValue = timeStampColByPheno.get(column.name()).getString(rowNum); - //If no timestamp, set to midnight - if (!dateTimeValue.isBlank() && !validDateTimeValue(dateTimeValue)) { - dateTimeValue += MIDNIGHT; - } - } - - // get the study year either referenced from the observation unit or listed explicitly on the import row - // TODO: handle this different workflows - String studyYear = hasAllReferenceUnitIds ? studyPIO.getBrAPIObject().getSeasons().get(0) : importRow.getEnvYear(); - String seasonDbId = sharedSeasonService.yearToSeasonDbId(studyYear, program.getId()); - fetchOrCreateObservationPIO( - program, - user, - importRow, - column, //column.name() gets phenotype name - rowNum, - dateTimeValue, - commit, - seasonDbId, - obsUnitPIO, - studyPIO, - referencedTraits - ); - } - } - } - - private Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { - Set ouDbIds = new HashSet<>(); - Set variableDbIds = new HashSet<>(); - Map variableNameByDbId = new HashMap<>(); - Map ouNameByDbId = new HashMap<>(); - Map studyNameByDbId = studyByNameNoScope.values() - .stream() - .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) - .map(PendingImportObject::getBrAPIObject) - .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); - - studyNameByDbId.keySet().forEach(studyDbId -> { - try { - brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { - if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { - ouDbIds.add(ou.getObservationUnitDbId()); - } - ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); - }); - } catch (ApiException e) { - throw new RuntimeException(e); - } - }); - - for (Trait referencedTrait : referencedTraits) { - variableDbIds.add(referencedTrait.getObservationVariableDbId()); - variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); - } - - List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); - - return existingObservations.stream() - .map(obs -> { - String studyName = studyNameByDbId.get(obs.getStudyDbId()); - String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); - String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); - - String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); - - return Map.entry(key, obs); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - private PendingImportObject fetchOrCreateTrialPIO( - Program program, - User user, - boolean commit, - ExperimentObservation importRow, - Supplier expNextVal - ) throws UnprocessableEntityException { - PendingImportObject trialPio; - - // use the prior trial if observation unit IDs are supplied - // TODO: handle multiple workflows - if (hasAllReferenceUnitIds) { - trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); - - // otherwise create a new trial, but there can be only one allowed - } else { - if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { - PendingImportObject envPio; - trialPio = trialByNameNoScope.get(importRow.getExpTitle()); - envPio = studyByNameNoScope.get(importRow.getEnv()); - - // creating new units for existing experiments and environments is not possible - if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && - (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ - throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); - } - } else if (!trialByNameNoScope.isEmpty()) { - throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); - } else { - UUID id = UUID.randomUUID(); - String expSeqValue = null; - if (commit) { - expSeqValue = expNextVal.get().toString(); - } - BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); - trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); - trialByNameNoScope.put(importRow.getExpTitle(), trialPio); - } - } - return trialPio; - } - - private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) throws UnprocessableEntityException { - PendingImportObject pio; - - // TODO: multiple workflows - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); - String name = String.format("Observation Dataset [%s-%s]", - program.getKey(), - trialPIO.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) - .getAsString()); - if (obsVarDatasetByName.containsKey(name)) { - pio = obsVarDatasetByName.get(name); - } else { - UUID id = UUID.randomUUID(); - BrAPIListDetails newDataset = importRow.constructDatasetDetails( - name, - id, - BRAPI_REFERENCE_SOURCE, - program, - trialPIO.getId().toString()); - pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); - trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); - if (ImportObjectState.EXISTING == trialPIO.getState()) { - trialPIO.setState(ImportObjectState.MUTATED); - } - obsVarDatasetByName.put(name, pio); - } - addObsVarsToDatasetDetails(pio, referencedTraits, program); - } - - private void fetchOrCreateLocationPIO(ExperimentObservation importRow) { - PendingImportObject pio; - // TODO: multiple workflows - String envLocationName = hasAllReferenceUnitIds ? - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName() : importRow.getEnvLocation(); - if (!locationByName.containsKey((importRow.getEnvLocation()))) { - ProgramLocation newLocation = new ProgramLocation(); - newLocation.setName(envLocationName); - pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); - this.locationByName.put(envLocationName, pio); - } - } - - private PendingImportObject fetchOrCreateStudyPIO( - Program program, - boolean commit, - String expSequenceValue, - ExperimentObservation importRow, - Supplier envNextVal - ) throws UnprocessableEntityException { - PendingImportObject pio; - // TODO: multiple workflows - if (hasAllReferenceUnitIds) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), - program.getKey() - ); - pio = studyByNameNoScope.get(studyName); - if (!commit){ - addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); - } - } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { - pio = studyByNameNoScope.get(importRow.getEnv()); - if (!commit){ - addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); - } - } else { - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); - UUID trialID = trialPIO.getId(); - UUID id = UUID.randomUUID(); - BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); - newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies - - // It is assumed that the study has only one season, And that the Years and not - // the dbId's are stored in getSeason() list. - String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season - if (commit) { - if(StringUtils.isNotBlank(year)) { - String seasonID = this.yearToSeasonDbId(year, program.getId()); - newStudy.setSeasons(Collections.singletonList(seasonID)); - } - } else { - addYearToStudyAdditionalInfo(program, newStudy, year); - } - - pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); - this.studyByNameNoScope.put(importRow.getEnv(), pio); - } - return pio; - } - - private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { - PendingImportObject pio; - String key = createObservationUnitKey(importRow); - // TODO: multiple workflows - if (hasAllReferenceUnitIds) { - pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); - } else if (observationUnitByNameNoScope.containsKey(key)) { - pio = observationUnitByNameNoScope.get(key); - } else { - String germplasmName = ""; - if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { - germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) - .getBrAPIObject() - .getGermplasmName(); - } - PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; - UUID trialID = trialPIO.getId(); - UUID datasetId = null; - if (commit) { - datasetId = UUID.fromString(trialPIO.getBrAPIObject() - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); - } - PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); - UUID studyID = studyPIO.getId(); - UUID id = UUID.randomUUID(); - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); - - // check for existing units if this is an existing study - if (studyPIO.getBrAPIObject().getStudyDbId() != null) { - List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); - List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); - if (matchingOU.isEmpty()) { - throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); - } else { - pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); - } - } else { - pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); - } - this.observationUnitByNameNoScope.put(key, pio); - } - return pio; - } - - private void fetchOrCreateObservationPIO(Program program, - User user, - ExperimentObservation importRow, - Column column, - Integer rowNum, - String timeStampValue, - boolean commit, - String seasonDbId, - PendingImportObject obsUnitPIO, - PendingImportObject studyPIO, - List referencedTraits) throws ApiException, UnprocessableEntityException { - PendingImportObject pio; - BrAPIObservation newObservation; - String variableName = column.name(); - String value = column.getString(rowNum); - String key; - - // TODO: multiple workflows - if (hasAllReferenceUnitIds) { - String unitName = obsUnitPIO.getBrAPIObject().getObservationUnitName(); - String studyName = studyPIO.getBrAPIObject().getStudyName(); - key = getObservationHash(studyName + unitName, variableName, studyName); - } else { - key = getImportObservationHash(importRow, variableName); - } - - if (existingObsByObsHash.containsKey(key)) { - if (!isObservationMatched(key, value, column, rowNum)){ - - // prior observation with updated value - newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); - if (!isValueMatched(key, value)){ - newObservation.setValue(value); - } else if (!StringUtils.isBlank(timeStampValue) && !isTimestampMatched(key, timeStampValue)) { - DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; - String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timeStampValue)); - newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); - } - pio = new PendingImportObject<>(ImportObjectState.MUTATED, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); - } else { - - // prior observation - pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(existingObsByObsHash.get(key), BrAPIObservation.class, program)); - } - - observationByHash.put(key, pio); - } else if (!this.observationByHash.containsKey(key)){ - - // new observation - // TODO: multiple workflows - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); - - UUID trialID = trialPIO.getId(); - UUID studyID = studyPIO.getId(); - UUID id = UUID.randomUUID(); - newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject(), commit, program, user, BRAPI_REFERENCE_SOURCE, trialID, studyID, obsUnitPIO.getId(), id); - //NOTE: Can't parse invalid timestamp value, so have to skip if invalid. - // Validation error should be thrown for offending value, but that doesn't happen until later downstream - if (timeStampValue != null && !timeStampValue.isBlank() && (validDateValue(timeStampValue) || validDateTimeValue(timeStampValue))) { - newObservation.setObservationTimeStamp(OffsetDateTime.parse(timeStampValue)); - } - - newObservation.setStudyDbId(studyPIO.getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations - - pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); - this.observationByHash.put(key, pio); - } - } - - */ - -} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java new file mode 100644 index 000000000..4ccdcd868 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java @@ -0,0 +1,464 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.JSON; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentSeasonService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.model.Trait; +import org.breedinginsight.model.User; +import org.breedinginsight.utilities.Utilities; +import tech.tablesaw.columns.Column; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Singleton +@Slf4j +public class ValidatePendingImportObjectsStep { + + private static final String BLANK_FIELD_EXPERIMENT = "Field is blank when creating a new experiment"; + private static final String ENV_LOCATION_MISMATCH = "All locations must be the same for a given environment"; + private static final String BLANK_FIELD_ENV = "Field is blank when creating a new environment"; + private static final String BLANK_FIELD_OBS = "Field is blank when creating new observations"; + private static final String ENV_YEAR_MISMATCH = "All years must be the same for a given environment"; + + private final ExperimentSeasonService experimentSeasonService; + private final Gson gson; + + @Inject + public ValidatePendingImportObjectsStep(ExperimentSeasonService experimentSeasonService) { + this.experimentSeasonService = experimentSeasonService; + this.gson = new JSON().getGson(); + } + + public ValidationErrors process(ImportContext input) { + ValidationErrors validationErrors = new ValidationErrors(); + + + return validationErrors; + + } + + private void prepareDataForValidation(List importRows, + PendingData pendingData, + List> phenotypeCols, + Map mappedBrAPIImport) { + + Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + Map> locationByName = pendingData.getLocationByName(); + Map> obsVarDatasetByName = pendingData.getObsVarDatasetByName(); + Map> existingGermplasmByGID = pendingData.getExistingGermplasmByGID(); + + + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); + List> observations = mappedImportRow.getObservations(); + String observationHash; + + // NOTE: Removed append/update workflow code + mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); + mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); + mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); + mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(ExperimentUtilities.createObservationUnitKey(importRow))); + mappedImportRow.setGermplasm(getGidPIO(pendingData, importRow)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(ExperimentUtilities.getImportObservationHash(importRow, ExperimentUtilities.getVariableNameFromColumn(column)))); + } + + mappedBrAPIImport.put(rowNum, mappedImportRow); + } + } + + private PendingImportObject getGidPIO(PendingData pendingData, ExperimentObservation importRow) { + + Map> existingGermplasmByGID = pendingData.getExistingGermplasmByGID(); + + if (existingGermplasmByGID.containsKey(importRow.getGid())) { + return existingGermplasmByGID.get(importRow.getGid()); + } + + return null; + } + + private ValidationErrors validateFields(List importRows, Map mappedBrAPIImport, List referencedTraits, Program program, + List> phenotypeCols, boolean commit, User user, + PendingData pendingData, + ProcessedPhenotypeData phenotypeData) { + //fetching any existing observations for any OUs in the import + CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); + ValidationErrors validationErrors = new ValidationErrors(); + + for ( Trait trait: referencedTraits) { + colVarMap.put(trait.getObservationVariableName(),trait); + } + Set uniqueStudyAndObsUnit = new HashSet<>(); + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); + // NOTE: validate Observations used by both workflows + if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. + validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); + } + validateTestOrCheck(importRow, validationErrors, rowNum); + validateConditionallyRequired(pendingData, validationErrors, rowNum, importRow, program, commit); + validateObservationUnits(pendingData, validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + validateObservations(pendingData, phenotypeData, validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); + } + + return validationErrors; + } + + private void validateGermplasm(ExperimentObservation importRow, ValidationErrors validationErrors, int rowNum, PendingImportObject germplasmPIO) { + // error if GID is not blank but GID does not already exist + if (StringUtils.isNotBlank(importRow.getGid()) && germplasmPIO == null) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.GERMPLASM_GID, "A non-existing GID", validationErrors, rowNum); + } + } + + private void validateTestOrCheck(ExperimentObservation importRow, ValidationErrors validationErrors, int rowNum) { + String testOrCheck = importRow.getTestOrCheck(); + if ( ! ( testOrCheck==null || testOrCheck.isBlank() + || "C".equalsIgnoreCase(testOrCheck) || "CHECK".equalsIgnoreCase(testOrCheck) + || "T".equalsIgnoreCase(testOrCheck) || "TEST".equalsIgnoreCase(testOrCheck) ) + ){ + ExperimentUtilities.addRowError(ExperimentObservation.Columns.TEST_CHECK, String.format("Invalid value (%s)", testOrCheck), validationErrors, rowNum) ; + } + } + + private void validateConditionallyRequired(PendingData pendingData, ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, Program program, boolean commit) { + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); + Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + + ImportObjectState expState = trialByNameNoScope.get(importRow.getExpTitle()) + .getState(); + ImportObjectState envState = studyByNameNoScope.get(importRow.getEnv()).getState(); + + String errorMessage = BLANK_FIELD_EXPERIMENT; + if (expState == ImportObjectState.EXISTING && envState == ImportObjectState.NEW) { + errorMessage = BLANK_FIELD_ENV; + } else if(expState == ImportObjectState.EXISTING && envState == ImportObjectState.EXISTING) { + errorMessage = BLANK_FIELD_OBS; + } + + if(expState == ImportObjectState.NEW || envState == ImportObjectState.NEW) { + validateRequiredCell(importRow.getGid(), ExperimentObservation.Columns.GERMPLASM_GID, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpTitle(), ExperimentObservation.Columns.EXP_TITLE,errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpUnit(), ExperimentObservation.Columns.EXP_UNIT, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpType(), ExperimentObservation.Columns.EXP_TYPE, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getEnv(), ExperimentObservation.Columns.ENV, errorMessage, validationErrors, rowNum); + if(validateRequiredCell(importRow.getEnvLocation(), ExperimentObservation.Columns.ENV_LOCATION, errorMessage, validationErrors, rowNum)) { + if(!Utilities.removeProgramKeyAndUnknownAdditionalData(studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getLocationName(), program.getKey()).equals(importRow.getEnvLocation())) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.ENV_LOCATION, ENV_LOCATION_MISMATCH, validationErrors, rowNum); + } + } + if(validateRequiredCell(importRow.getEnvYear(), ExperimentObservation.Columns.ENV_YEAR, errorMessage, validationErrors, rowNum)) { + String studyYear = StringUtils.defaultString(studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getSeasons().get(0) ); + String rowYear = importRow.getEnvYear(); + if(commit) { + rowYear = experimentSeasonService.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); + } + if(StringUtils.isNotBlank(studyYear) && !studyYear.equals(rowYear)) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.ENV_YEAR, ENV_YEAR_MISMATCH, validationErrors, rowNum); + } + } + validateRequiredCell(importRow.getExpUnitId(), ExperimentObservation.Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpReplicateNo(), ExperimentObservation.Columns.REP_NUM, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpBlockNo(), ExperimentObservation.Columns.BLOCK_NUM, errorMessage, validationErrors, rowNum); + + if(StringUtils.isNotBlank(importRow.getObsUnitID())) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); + } + } else { + //Check if existing environment. If so, ObsUnitId must be assigned + validateRequiredCell( + importRow.getObsUnitID(), + ExperimentObservation.Columns.OBS_UNIT_ID, + ExperimentUtilities.MISSING_OBS_UNIT_ID_ERROR, + validationErrors, + rowNum + ); + } + } + + private boolean validateRequiredCell(String value, String columnHeader, String errorMessage, ValidationErrors validationErrors, int rowNum) { + if (StringUtils.isBlank(value)) { + ExperimentUtilities.addRowError(columnHeader, errorMessage, validationErrors, rowNum); + return false; + } + return true; + } + + private void validateObservationUnits( + PendingData pendingData, + ValidationErrors validationErrors, + Set uniqueStudyAndObsUnit, + int rowNum, + ExperimentObservation importRow) { + Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); + + validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + + String key = ExperimentUtilities.createObservationUnitKey(importRow); + PendingImportObject ouPIO = observationUnitByNameNoScope.get(key); + if(ouPIO.getState() == ImportObjectState.NEW && StringUtils.isNotBlank(importRow.getObsUnitID())) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "Could not find observation unit by ObsUnitDBID", validationErrors, rowNum); + } + + validateGeoCoordinates(validationErrors, rowNum, importRow); + } + + /** + * Validate that the observation unit is unique within a study. + *
+ * SIDE EFFECTS: validationErrors and uniqueStudyAndObsUnit can be modified. + * + * @param validationErrors can be modified as a side effect. + * @param uniqueStudyAndObsUnit can be modified as a side effect. + * @param rowNum counter that is always two less the file row being validated + * @param importRow the data row being validated + */ + private void validateUniqueObsUnits( + ValidationErrors validationErrors, + Set uniqueStudyAndObsUnit, + int rowNum, + ExperimentObservation importRow) { + String envIdPlusStudyId = ExperimentUtilities.createObservationUnitKey(importRow); + if (uniqueStudyAndObsUnit.contains(envIdPlusStudyId)) { + String errorMessage = String.format("The ID (%s) is not unique within the environment(%s)", importRow.getExpUnitId(), importRow.getEnv()); + ExperimentUtilities.addRowError(ExperimentObservation.Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); + } else { + uniqueStudyAndObsUnit.add(envIdPlusStudyId); + } + } + + private void validateGeoCoordinates(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow) { + + String lat = importRow.getLatitude(); + String lon = importRow.getLongitude(); + String elevation = importRow.getElevation(); + + // If any of Lat, Long, or Elevation are provided, Lat and Long must both be provided. + if (StringUtils.isNotBlank(lat) || StringUtils.isNotBlank(lon) || StringUtils.isNotBlank(elevation)) { + if (StringUtils.isBlank(lat)) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.LAT, "Latitude must be provided for complete coordinate specification", validationErrors, rowNum); + } + if (StringUtils.isBlank(lon)) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.LONG, "Longitude must be provided for complete coordinate specification", validationErrors, rowNum); + } + } + + // Validate coordinate values + boolean latBadValue = false; + boolean lonBadValue = false; + boolean elevationBadValue = false; + double latDouble; + double lonDouble; + double elevationDouble; + + // Only check latitude format if not blank since already had previous error + if (StringUtils.isNotBlank(lat)) { + try { + latDouble = Double.parseDouble(lat); + if (latDouble < -90 || latDouble > 90) { + latBadValue = true; + } + } catch (NumberFormatException e) { + latBadValue = true; + } + } + + // Only check longitude format if not blank since already had previous error + if (StringUtils.isNotBlank(lon)) { + try { + lonDouble = Double.parseDouble(lon); + if (lonDouble < -180 || lonDouble > 180) { + lonBadValue = true; + } + } catch (NumberFormatException e) { + lonBadValue = true; + } + } + + if (StringUtils.isNotBlank(elevation)) { + try { + elevationDouble = Double.parseDouble(elevation); + } catch (NumberFormatException e) { + elevationBadValue = true; + } + } + + if (latBadValue) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.LAT, "Invalid Lat value (expected range -90 to 90)", validationErrors, rowNum); + } + + if (lonBadValue) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.LONG, "Invalid Long value (expected range -180 to 180)", validationErrors, rowNum); + } + + if (elevationBadValue) { + ExperimentUtilities.addRowError(ExperimentObservation.Columns.LONG, "Invalid Elevation value (numerals expected)", validationErrors, rowNum); + } + + } + + + private void validateObservations(PendingData pendingData, + ProcessedPhenotypeData phenotypeData, + ValidationErrors validationErrors, + int rowNum, + ExperimentObservation importRow, + List> phenotypeCols, + CaseInsensitiveMap colVarMap, + boolean commit, + User user) { + + Map existingObsByObsHash = pendingData.getExistingObsByObsHash(); + Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); + + phenotypeCols.forEach(phenoCol -> { + String importHash; + String importObsValue = phenoCol.getString(rowNum); + + // NOTE: removed append / update specifc code + importHash = ExperimentUtilities.getImportObservationHash(importRow, phenoCol.name()); + + // error if import observation data already exists and user has not selected to overwrite + if(commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && + existingObsByObsHash.containsKey(importHash) && + StringUtils.isNotBlank(phenoCol.getString(rowNum)) && + !existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { + ExperimentUtilities.addRowError( + phenoCol.name(), + String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), + validationErrors, rowNum + ); + + // preview case where observation has already been committed and the import row ObsVar data differs from what + // had been saved prior to import + } else if (existingObsByObsHash.containsKey(importHash) && !ExperimentUtilities.isObservationMatched(phenotypeData, pendingData, importHash, importObsValue, phenoCol, rowNum)) { + + // different data means validations still need to happen + // TODO consider moving these two calls into a separate method since called twice together + ExperimentUtilities.validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); + + //Timestamp validation + if(timeStampColByPheno.containsKey(phenoCol.name())) { + Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); + ExperimentUtilities.validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); + } + + // add a change log entry when updating the value of an observation + // only will update and thereby need change log entry if no error + if (commit && (!validationErrors.hasErrors())) { + BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; + String prior = ""; + if (ExperimentUtilities.isValueMatched(pendingData, importHash, importObsValue)) { + prior.concat(existingObsByObsHash.get(importHash).getValue()); + } + if (timeStampColByPheno.containsKey(phenoCol.name()) && ExperimentUtilities.isTimestampMatched(pendingData, importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { + prior = prior.isEmpty() ? prior : prior.concat(" "); + prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); + } + ChangeLogEntry change = new ChangeLogEntry(prior, + reason, + user.getId(), + timestamp + ); + + // create the changelog field in additional info if it does not already exist + if (pendingObservation.getAdditionalInfo().isJsonNull()) { + pendingObservation.setAdditionalInfo(new JsonObject()); + pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + + if (pendingObservation.getAdditionalInfo() != null && !pendingObservation.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { + pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + + // add a new entry to the changelog + pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); + } + + // preview case where observation has already been committed and import ObsVar data is the + // same as has been committed prior to import + } else if(ExperimentUtilities.isObservationMatched(phenotypeData, pendingData, importHash, importObsValue, phenoCol, rowNum)) { + BrAPIObservation existingObs = existingObsByObsHash.get(importHash); + existingObs.setObservationVariableName(phenoCol.name()); + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + observationByHash.get(importHash).setBrAPIObject(existingObs); + + // preview case where observation has already been committed and import ObsVar data is empty prior to import + } else if(!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + } else { + ExperimentUtilities.validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); + + //Timestamp validation + if(timeStampColByPheno.containsKey(phenoCol.name())) { + Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); + ExperimentUtilities.validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); + } + } + }); + } + + + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java index 5fd67a023..c365cbf6d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentSeasonService.java @@ -16,6 +16,8 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment.services; +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; @@ -30,7 +32,8 @@ import java.util.Map; import java.util.UUID; -@Singleton +// reset cache across uses by creating new instance each time this service is injected +@Prototype @Slf4j public class ExperimentSeasonService { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java index af2bcb29e..d12ff2509 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentValidateService.java @@ -113,4 +113,6 @@ private List fetchFileTraits(UUID programId, Collection varNames) throw new InternalServerException(e.toString(), e); } } + + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java deleted file mode 100644 index 9ca689447..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/PendingImportObjectPopulator.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.services.processors.experiment.services; - -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; - -import java.math.BigInteger; -import java.util.function.Supplier; - -public interface PendingImportObjectPopulator { - - PendingImportObject populateTrial( - ImportContext importContext, - PendingData pendingData, - ExperimentObservation importRow, - Supplier expNextVal - ) throws UnprocessableEntityException; - - PendingImportObject populateStudy( - ImportContext importContext, - Supplier expSeqValue, - ExperimentObservation importRow, - Supplier envNextVal); -} From c8d1cb6ef4a7d46074e249d00a692898fdb1bf43 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:24:47 -0400 Subject: [PATCH 32/50] Building --- .../ExperimentImportService.java | 2 + .../model/workflow/ImportContext.java | 2 +- .../model/workflow/ProcessedData.java | 8 +- .../AppendOverwritePhenotypesWorkflow.java | 3 +- .../workflow/CreateNewExperimentWorkflow.java | 106 ++++++++++++-- .../steps/CommitPendingImportObjectsStep.java | 3 +- ...ulateExistingPendingImportObjectsStep.java | 3 +- .../PopulateNewPendingImportObjectsStep.java | 134 ++++++++---------- .../ValidatePendingImportObjectsStep.java | 35 +++-- .../CreateNewEnvironmentWorkflow.java | 3 +- 10 files changed, 192 insertions(+), 107 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index fededeea1..616b6fa58 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -34,6 +34,7 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import java.util.HashMap; import java.util.List; @Singleton @@ -85,6 +86,7 @@ public ImportPreviewResponse process(ImportServiceContext context) .data(context.getData()) .commit(context.isCommit()) .upload(context.getUpload()) + .program(context.getProgram()) .build(); return context.getWorkflow().process(importContext); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java index 109c300cc..f9205cb87 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java @@ -40,7 +40,7 @@ public class ImportContext { private ImportUpload upload; private List importRows; // TODO: move this out potentially - private Map mappedBrAPIImport; + //private Map mappedBrAPIImport; private Table data; private Program program; private User user; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java index 73302bdb9..677f35469 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -18,19 +18,19 @@ package org.breedinginsight.brapps.importer.model.workflow; import lombok.*; -import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; -import java.util.ArrayList; -import java.util.List; import java.util.Map; @Data @ToString @NoArgsConstructor public class ProcessedData { - private Map statistics; + // TODO: remove this, already in ImportPreviewResponse + //private Map statistics; + // TODO private Map mappedBrAPIImport; + private ImportPreviewResponse importPreviewResponse; } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index 21df19be6..124a3a0f9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -18,6 +18,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; import org.breedinginsight.brapps.importer.model.workflow.Workflow; @@ -33,7 +34,7 @@ @Named("AppendOverwritePhenotypesWorkflow") public class AppendOverwritePhenotypesWorkflow implements Workflow { @Override - public ProcessedData process(ImportContext context) { + public ImportPreviewResponse process(ImportContext context) { // TODO return null; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index e32a5c29c..343b4475d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -23,17 +23,22 @@ import lombok.extern.slf4j.Slf4j; import lombok.val; import org.apache.commons.lang3.StringUtils; +import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.ImportStatusService; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.CommitPendingImportObjectsStep; @@ -46,6 +51,7 @@ import javax.inject.Inject; import javax.inject.Named; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -86,11 +92,13 @@ public ImportPreviewResponse process(ImportContext context) throws Exception { ImportUpload upload = context.getUpload(); boolean commit = context.isCommit(); + List importRows = context.getImportRows(); // Make sure the file does not contain obs unit ids before proceeding if (containsObsUnitIDs(context)) { // TODO: get file name - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Error detected in file, XXX.xls. ObsUnitIDs are detected. Import cannot proceed"); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Error detected in file, " + + upload.getUploadFileName() + ". ObsUnitIDs are detected. Import cannot proceed"); } statusService.updateMessage(upload, "Checking existing experiment objects in brapi service and mapping data"); @@ -98,14 +106,15 @@ public ImportPreviewResponse process(ImportContext context) throws Exception { ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(context); ProcessContext processContext = populateExistingPendingImportObjectsStep.process(context, phenotypeData); ProcessedData processedData = populateNewPendingImportObjectsStep.process(processContext, phenotypeData); - ValidationErrors validationErrors = validatePendingImportObjectsStep.process(context); + ValidationErrors validationErrors = validatePendingImportObjectsStep.process(context, processContext.getPendingData(), phenotypeData, processedData); // short circuit if there were validation errors if (validationErrors.hasErrors()) { throw new ValidatorException(validationErrors); } - ImportPreviewResponse response = buildImportPreviewResponse(processedData, upload); + // TODO: move to experiment import service + ImportPreviewResponse response = buildImportPreviewResponse(importRows, processContext.getPendingData(), processedData, upload); statusService.updateMappedData(upload, response, "Finished mapping data to brapi objects"); @@ -116,7 +125,7 @@ public ImportPreviewResponse process(ImportContext context) throws Exception { } // commit data - long totalObjects = getNewObjectCount(processedData); + long totalObjects = getNewObjectCount(response); statusService.startUpload(upload, totalObjects, "Starting upload to brapi service"); statusService.updateMessage(upload, "Creating new experiment objects in brapi service"); @@ -138,10 +147,11 @@ public String getName() { } // TODO: move to shared area - private ImportPreviewResponse buildImportPreviewResponse(ProcessedData processedData, + private ImportPreviewResponse buildImportPreviewResponse(List importRows, PendingData pendingData, ProcessedData processedData, ImportUpload upload) { - Map statistics = processedData.getStatistics(); + Map mappedBrAPIImport = processedData.getMappedBrAPIImport(); + Map statistics = generateStatisticsMap(pendingData, importRows); ImportPreviewResponse response = new ImportPreviewResponse(); response.setStatistics(statistics); @@ -152,10 +162,10 @@ private ImportPreviewResponse buildImportPreviewResponse(ProcessedData processed } // TODO: move to shared area - private long getNewObjectCount(ProcessedData processedData) { + private long getNewObjectCount(ImportPreviewResponse response) { // get total number of new brapi objects to create long totalObjects = 0; - for (ImportPreviewStatistics stats : processedData.getStatistics().values()) { + for (ImportPreviewStatistics stats : response.getStatistics().values()) { totalObjects += stats.getNewObjectCount(); } return totalObjects; @@ -169,5 +179,85 @@ private boolean containsObsUnitIDs(ImportContext importContext) { return StringUtils.isNotBlank(expRow.getObsUnitID()); }); } + + // TODO: move to shared area: experiment import service + private Map generateStatisticsMap(PendingData pendingData, List importRows) { + // Data for stats. + HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names + HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's + HashSet gidCounter = new HashSet<>(); // set of unique GID's + + Map> observationByHash = pendingData.getObservationByHash(); + + for (BrAPIImport row : importRows) { + ExperimentObservation importRow = (ExperimentObservation) row; + // Collect date for stats. + addIfNotNull(environmentNameCounter, importRow.getEnv()); + addIfNotNull(obsUnitsIDCounter, ExperimentUtilities.createObservationUnitKey(importRow)); + addIfNotNull(gidCounter, importRow.getGid()); + } + + int numNewObservations = Math.toIntExact( + observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + int numExistingObservations = Math.toIntExact( + observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.EXISTING && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + int numMutatedObservations = Math.toIntExact( + observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() + .newObjectCount(environmentNameCounter.size()) + .build(); + ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() + .newObjectCount(obsUnitsIDCounter.size()) + .build(); + ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() + .newObjectCount(gidCounter.size()) + .build(); + ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder() + .newObjectCount(numNewObservations) + .build(); + ImportPreviewStatistics existingObservationStats = ImportPreviewStatistics.builder() + .newObjectCount(numExistingObservations) + .build(); + ImportPreviewStatistics mutatedObservationStats = ImportPreviewStatistics.builder() + .newObjectCount(numMutatedObservations) + .build(); + + return Map.of( + "Environments", environmentStats, + "Observation_Units", obdUnitStats, + "GIDs", gidStats, + "Observations", observationStats, + "Existing_Observations", existingObservationStats, + "Mutated_Observations", mutatedObservationStats + ); + } + + // TODO: move to common area + private void addIfNotNull(HashSet set, String setValue) { + if (setValue != null) { + set.add(setValue); + } + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index fa02c76ed..12e6bcc99 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -95,6 +95,7 @@ public void process(ProcessContext processContext, ProcessedData processedData) ImportUpload upload = importContext.getUpload(); Program program = importContext.getProgram(); + Map mappedBrAPIImport = processedData.getMappedBrAPIImport(); Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); @@ -159,7 +160,7 @@ public void process(ProcessContext processContext, ProcessedData processedData) .setLocationDbId(createdLocation.getLocationDbId()); } - updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); + updateStudyDependencyValues(pendingData, mappedBrAPIImport, program.getKey()); List createdStudies = brAPIStudyDAO.createBrAPIStudies(newStudies, program.getId(), upload); // set the DbId to the for each newly created study diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index 22799a025..c5196e662 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -120,6 +120,7 @@ public ProcessContext process(ImportContext input, ProcessedPhenotypeData phenot .obsVarDatasetByName(obsVarDatasetByName) .existingGermplasmByGID(existingGermplasmByGID) .existingObsByObsHash(existingObsByObsHash) + .observationByHash(new HashMap<>()) .build(); return ProcessContext.builder() @@ -539,7 +540,7 @@ private Map fetchExistingObservations(List refe String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); - String key = ExperimentUtilities.getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); + String key = ExperimentUtilities.getObservationHash(ExperimentUtilities.createObservationUnitKey(studyName, ouName), variableName, studyName); return Map.entry(key, obs); }) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index 2d22895fc..a471b84f3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -16,12 +16,13 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; +import com.google.gson.Gson; import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpStatus; import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.apache.xmlbeans.impl.xb.xsdschema.ImportDocument; +import org.brapi.client.v2.JSON; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; @@ -77,6 +78,7 @@ public class PopulateNewPendingImportObjectsStep { private final BrAPIObservationDAO brAPIObservationDAO; private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final DSLContext dsl; + private final Gson gson; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @@ -92,6 +94,7 @@ public PopulateNewPendingImportObjectsStep(ExperimentValidateService experimentV this.brAPIObservationDAO = brAPIObservationDAO; this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.dsl = dsl; + this.gson = new JSON().getGson(); } public ProcessedData process(ProcessContext context, ProcessedPhenotypeData phenotypeData) @@ -131,13 +134,11 @@ private void populatePendingImportObjects(ProcessContext processContext, for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - populateIndependentVariablePIOsForRow(importContext, phenotypeData, pendingData, importRow, expNextVal, envNextVal); + PendingImportObjectData pioData = populateIndependentVariablePIOsForRow(importContext, + phenotypeData, pendingData, importRow, expNextVal, envNextVal); - // ... (Common logic from the original method) - - PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); - - processObservations(importContext, phenotypeData, importRow, rowNum, commit, importRow.getEnvYear(), obsUnitPIO, studyPIO); + processObservations(importContext, + phenotypeData, pendingData, importRow, rowNum, commit, pioData.getObsUnitPIO(), pioData.getStudyPIO()); } } @@ -247,10 +248,14 @@ private PendingImportObjectData populateIndependentVariablePIOsForRow(ImportCont .build(); } - private void processObservations(ImportContext importContext, ProcessedPhenotypeData phenotypeData, ExperimentObservation importRow, - int rowNum, boolean commit, String studyYear, - PendingImportObject obsUnitPIO, - PendingImportObject studyPIO) + // TODO: some reusable stuff between workflows that could potentially be broken out + private void processObservations(ImportContext importContext, + ProcessedPhenotypeData phenotypeData, + PendingData pendingData, + ExperimentObservation importRow, + int rowNum, boolean commit, + PendingImportObject obsUnitPIO, + PendingImportObject studyPIO) throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { Program program = importContext.getProgram(); User user = importContext.getUser(); @@ -259,56 +264,35 @@ private void processObservations(ImportContext importContext, ProcessedPhenotype List referencedTraits = phenotypeData.getReferencedTraits(); for (Column column : phenotypeCols) { - // ... (Logic for processing observations) - } - } - - // TODO: move common code out - private void initNewBrapiData(ImportContext importContext, ProcessedPhenotypeData phenotypeData) - throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { - - Program program = importContext.getProgram(); - List importRows = importContext.getImportRows(); - User user = importContext.getUser(); - boolean commit = importContext.isCommit(); - Map existingObsByObsHash; - - List referencedTraits = phenotypeData.getReferencedTraits(); - List> phenotypeCols = phenotypeData.getPhenotypeCols(); - Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); - - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - - - for (Column column : phenotypeCols) { - //If associated timestamp column, add - String dateTimeValue = null; - if (timeStampColByPheno.containsKey(column.name())) { - dateTimeValue = timeStampColByPheno.get(column.name()).getString(rowNum); - //If no timestamp, set to midnight - if (!dateTimeValue.isBlank() && !validDateTimeValue(dateTimeValue)) { - dateTimeValue += MIDNIGHT; - } + //If associated timestamp column, add + String dateTimeValue = null; + if (timeStampColByPheno.containsKey(column.name())) { + dateTimeValue = timeStampColByPheno.get(column.name()).getString(rowNum); + //If no timestamp, set to midnight + if (!dateTimeValue.isBlank() && !validDateTimeValue(dateTimeValue)) { + dateTimeValue += MIDNIGHT; } - - // get the study year either referenced from the observation unit or listed explicitly on the import row - // TODO: handle this different workflows - String studyYear = hasAllReferenceUnitIds ? studyPIO.getBrAPIObject().getSeasons().get(0) : importRow.getEnvYear(); - String seasonDbId = experimentSeasonService.yearToSeasonDbId(studyYear, program.getId()); - fetchOrCreateObservationPIO( - program, - user, - importRow, - column, //column.name() gets phenotype name - rowNum, - dateTimeValue, - commit, - seasonDbId, - obsUnitPIO, - studyPIO, - referencedTraits - ); } + + // get the study year either referenced from the observation unit or listed explicitly on the import row + // NOTE: removed append / update code + String studyYear = importRow.getEnvYear(); + String seasonDbId = experimentSeasonService.yearToSeasonDbId(studyYear, program.getId()); + fetchOrCreateObservationPIO( + phenotypeData, + pendingData, + program, + user, + importRow, + column, //column.name() gets phenotype name + rowNum, + dateTimeValue, + commit, + seasonDbId, + obsUnitPIO, + studyPIO, + referencedTraits + ); } } @@ -563,7 +547,9 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Import return pio; } - private void fetchOrCreateObservationPIO(Program program, + private void fetchOrCreateObservationPIO(ProcessedPhenotypeData phenotypeData, + PendingData pendingData, + Program program, User user, ExperimentObservation importRow, Column column, @@ -579,24 +565,21 @@ private void fetchOrCreateObservationPIO(Program program, String variableName = column.name(); String value = column.getString(rowNum); String key; + Map existingObsByObsHash = pendingData.getExistingObsByObsHash(); + Map> observationByHash = pendingData.getObservationByHash(); + Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); - // TODO: multiple workflows - if (hasAllReferenceUnitIds) { - String unitName = obsUnitPIO.getBrAPIObject().getObservationUnitName(); - String studyName = studyPIO.getBrAPIObject().getStudyName(); - key = getObservationHash(studyName + unitName, variableName, studyName); - } else { - key = getImportObservationHash(importRow, variableName); - } + // NOTE: removed append / update + key = getImportObservationHash(importRow, variableName); if (existingObsByObsHash.containsKey(key)) { - if (!isObservationMatched(key, value, column, rowNum)){ + if (!isObservationMatched(phenotypeData, pendingData, key, value, column, rowNum)){ // prior observation with updated value newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); - if (!isValueMatched(key, value)){ + if (!isValueMatched(pendingData, key, value)){ newObservation.setValue(value); - } else if (!StringUtils.isBlank(timeStampValue) && !isTimestampMatched(key, timeStampValue)) { + } else if (!StringUtils.isBlank(timeStampValue) && !isTimestampMatched(pendingData, key, timeStampValue)) { DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timeStampValue)); newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); @@ -609,12 +592,11 @@ private void fetchOrCreateObservationPIO(Program program, } observationByHash.put(key, pio); - } else if (!this.observationByHash.containsKey(key)){ + } else if (!observationByHash.containsKey(key)){ // new observation - // TODO: multiple workflows - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + // NOTE: removed append / update code + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); UUID studyID = studyPIO.getId(); @@ -629,7 +611,7 @@ private void fetchOrCreateObservationPIO(Program program, newObservation.setStudyDbId(studyPIO.getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); - this.observationByHash.put(key, pio); + observationByHash.put(key, pio); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java index 4ccdcd868..99fbc12eb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java @@ -39,6 +39,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; @@ -54,10 +55,7 @@ import javax.inject.Singleton; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; @Singleton @Slf4j @@ -78,18 +76,27 @@ public ValidatePendingImportObjectsStep(ExperimentSeasonService experimentSeason this.gson = new JSON().getGson(); } - public ValidationErrors process(ImportContext input) { - ValidationErrors validationErrors = new ValidationErrors(); + public ValidationErrors process(ImportContext importContext, PendingData pendingData, ProcessedPhenotypeData phenotypeData, ProcessedData processedData) { + //Map mappedBrAPIImport = processedData.getMappedBrAPIImport(); + List importRows = importContext.getImportRows(); + List> phenotypeCols = phenotypeData.getPhenotypeCols(); + Program program = importContext.getProgram(); + List referencedTraits = phenotypeData.getReferencedTraits(); + boolean commit = importContext.isCommit(); + User user = importContext.getUser(); + Map mappedBrAPIImport = prepareDataForValidation(importRows, pendingData, phenotypeCols); + ValidationErrors validationErrors = validateFields(importRows, mappedBrAPIImport, referencedTraits, program, phenotypeCols, commit, user, pendingData, phenotypeData); + processedData.setMappedBrAPIImport(mappedBrAPIImport); return validationErrors; - } - private void prepareDataForValidation(List importRows, + private Map prepareDataForValidation(List importRows, PendingData pendingData, - List> phenotypeCols, - Map mappedBrAPIImport) { + List> phenotypeCols) { + + Map mappedBrAPIImport = new HashMap<>(); Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); @@ -97,7 +104,7 @@ private void prepareDataForValidation(List importRows, Map> locationByName = pendingData.getLocationByName(); Map> obsVarDatasetByName = pendingData.getObsVarDatasetByName(); Map> existingGermplasmByGID = pendingData.getExistingGermplasmByGID(); - + Map> observationByHash = pendingData.getObservationByHash(); for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); @@ -121,6 +128,8 @@ private void prepareDataForValidation(List importRows, mappedBrAPIImport.put(rowNum, mappedImportRow); } + + return mappedBrAPIImport; } private PendingImportObject getGidPIO(PendingData pendingData, ExperimentObservation importRow) { @@ -368,6 +377,7 @@ private void validateObservations(PendingData pendingData, Map existingObsByObsHash = pendingData.getExistingObsByObsHash(); Map> timeStampColByPheno = phenotypeData.getTimeStampColByPheno(); + Map> observationByHash = pendingData.getObservationByHash(); phenotypeCols.forEach(phenoCol -> { String importHash; @@ -458,7 +468,4 @@ private void validateObservations(PendingData pendingData, } }); } - - - } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java index 5776ab59b..05269964f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java @@ -18,6 +18,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; import org.breedinginsight.brapps.importer.model.workflow.Workflow; @@ -33,7 +34,7 @@ @Named("CreateNewEnvironmentWorkflow") public class CreateNewEnvironmentWorkflow implements Workflow { @Override - public ProcessedData process(ImportContext context) { + public ImportPreviewResponse process(ImportContext context) { // TODO return null; } From d57e4aced1386de7d3d3fdafb7d54043e686358d Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:29:06 -0400 Subject: [PATCH 33/50] Renamed to have github actions run --- ...riment_workflows.sql => V1.23.0__add_experiment_workflows.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V1.22.0__add_experiment_workflows.sql => V1.23.0__add_experiment_workflows.sql} (100%) diff --git a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql similarity index 100% rename from src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql rename to src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql From 79189ba768ad3219c57ea950b04a543409e6b77f Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 18 Jun 2024 14:18:23 -0400 Subject: [PATCH 34/50] Messing with dependency issue --- .../experimentObservation/ExperimentImportService.java | 9 +++++++++ .../steps/PopulateNewPendingImportObjectsStep.java | 8 +------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 149da0e77..013d55012 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.model.imports.DomainImportService; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; @@ -26,12 +27,14 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import java.util.List; @Singleton @Slf4j public class ExperimentImportService extends DomainImportService { private final String IMPORT_TYPE_ID = "ExperimentImport"; + private final ExperimentWorkflowNavigator workflowNavigator; // TODO: delete processor fields once WorkflowNavigator is used @Inject @@ -40,6 +43,7 @@ public ExperimentImportService(Provider experimentProcessor ExperimentWorkflowNavigator workflowNavigator) { super(experimentProcessorProvider, processorManagerProvider, workflowNavigator); + this.workflowNavigator = workflowNavigator; } @Override @@ -51,5 +55,10 @@ public ExperimentObservation getImportClass() { public String getImportTypeId() { return IMPORT_TYPE_ID; } + + @Override + public List getWorkflows() throws Exception{ + return workflowNavigator.getWorkflows(); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index a471b84f3..f5ec2cc96 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -73,9 +73,7 @@ @Slf4j public class PopulateNewPendingImportObjectsStep { - private final ExperimentValidateService experimentValidateService; private final ExperimentSeasonService experimentSeasonService; - private final BrAPIObservationDAO brAPIObservationDAO; private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final DSLContext dsl; private final Gson gson; @@ -84,14 +82,10 @@ public class PopulateNewPendingImportObjectsStep { private String BRAPI_REFERENCE_SOURCE; @Inject - public PopulateNewPendingImportObjectsStep(ExperimentValidateService experimentValidateService, - ExperimentSeasonService experimentSeasonService, - BrAPIObservationDAO brAPIObservationDAO, + public PopulateNewPendingImportObjectsStep(ExperimentSeasonService experimentSeasonService, BrAPIObservationUnitDAO brAPIObservationUnitDAO, DSLContext dsl) { - this.experimentValidateService = experimentValidateService; this.experimentSeasonService = experimentSeasonService; - this.brAPIObservationDAO = brAPIObservationDAO; this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.dsl = dsl; this.gson = new JSON().getGson(); From 0ef8cd5d21a559ea886683b3109dcea422acd147 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 14:57:51 -0400 Subject: [PATCH 35/50] Removed ununsed fileImportService injection and resolved DI issue --- .../brapps/importer/services/FileImportService.java | 5 +++++ .../brapps/importer/services/FileMappingUtil.java | 8 +------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 0c2a9309d..da5a68e10 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -576,6 +576,11 @@ public List getWorkflowsForSystemMapping(UUID mappingId) throws .orElseThrow(() -> new DoesNotExistException("Cannot find mapping config associated with upload.")); BrAPIImportService importService = configManager.getImportServiceById(mappingConfig.getImportTypeId()) .orElseThrow(() -> new DoesNotExistException("Config with that id does not exist")); + // NOTE: + // this is creating a workflow navigator to call getWorkflows + // workflowNavigator.getWorkflows(); + // getWorkflows is creating the ExperimentWorkflow which is injecting the file import service and + // resulting in a circular dependency return importService.getWorkflows(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java index cb93daf48..8d3ed8146 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java @@ -32,13 +32,7 @@ @Singleton public class FileMappingUtil { - public static final String EXPERIMENT_TEMPLATE_NAME = "ExperimentsTemplateMap"; - private FileImportService fileImportService; - - - @Inject - public FileMappingUtil(FileImportService fileImportService) { - this.fileImportService = fileImportService; + public FileMappingUtil() { } // Returns a list of integers to identify the target row of the relationship. -1 if no relationship was found From 1acd04bd343865d81edbf02d2af56b05bf04f8e7 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:23:25 -0400 Subject: [PATCH 36/50] Added in BI-2128 change --- .../workflow/steps/PopulateNewPendingImportObjectsStep.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index f5ec2cc96..ddb4193ab 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -567,7 +567,9 @@ private void fetchOrCreateObservationPIO(ProcessedPhenotypeData phenotypeData, key = getImportObservationHash(importRow, variableName); if (existingObsByObsHash.containsKey(key)) { - if (!isObservationMatched(phenotypeData, pendingData, key, value, column, rowNum)){ + // NOTE: BI-2128 change added after refactor branch + // Update observation value only if it is changed and new value is not blank. + if (!isObservationMatched(phenotypeData, pendingData, key, value, column, rowNum) && StringUtils.isNotBlank(value)){ // prior observation with updated value newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); From 6a4043d2202adfa7c97e06d65bbc4d14c5045fc3 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:36:42 -0400 Subject: [PATCH 37/50] Updated create only tests to use create workflow --- .../importer/ExperimentFileImportTest.java | 239 ++++++++++++------ .../brapps/importer/ImportTestUtils.java | 95 ++++++- 2 files changed, 243 insertions(+), 91 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 48bffd4c2..fbb02435e 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -24,6 +24,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.client.RxHttpClient; import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.netty.cookies.NettyCookie; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; import lombok.SneakyThrows; @@ -77,6 +78,7 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import static io.micronaut.http.HttpRequest.GET; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.*; @@ -142,9 +144,11 @@ public class ExperimentFileImportTest extends BrAPITest { private BrAPISeasonDAO seasonDAO; private Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) - (json, type, context) -> OffsetDateTime.parse(json.getAsString())) - .registerTypeAdapter(BrAPIPagination.class, new PaginationTypeAdapter()) - .create(); + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .registerTypeAdapter(BrAPIPagination.class, new PaginationTypeAdapter()) + .create(); + + private String newExperimentWorkflowId; @BeforeAll public void setup() { @@ -153,7 +157,27 @@ public void setup() { mappingId = (String) setupObjects.get("mappingId"); testUser = (BiUserEntity) setupObjects.get("testUser"); securityFp = (FannyPack) setupObjects.get("securityFp"); + newExperimentWorkflowId = getNewExperimentWorkflowId(); + } + + /** + * TODO: assumes new workflow is first in list, doesn't look at position property, would be more robust to + * look at that instead of assuming order + * @return + */ + public String getNewExperimentWorkflowId() { + // GET /import/mappings{?importName} + Flowable> call = client.exchange( + GET("/import/mappings/"+mappingId+"/workflows").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + + return JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); } /* @@ -193,16 +217,20 @@ public void importNewExpNewLocNoObsSuccess() { validRow.put(Columns.COLUMN, "1"); validRow.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String workflowId = "new-experiment"; + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId, newExperimentWorkflowId); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + // TODO: remove this + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); @@ -251,16 +279,18 @@ public void importNewExpMultiNewEnvSuccess() { secondEnv.put(Columns.COLUMN, "1"); secondEnv.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(2, previewRows.size()); JsonObject firstRow = previewRows.get(0).getAsJsonObject(); @@ -298,7 +328,8 @@ public void importExistingExpAndEnvErrorMessage() { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + JsonObject expResult = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + //JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); Map dupExp = new HashMap<>(); dupExp.put(Columns.GERMPLASM_GID, "1"); @@ -315,16 +346,17 @@ public void importExistingExpAndEnvErrorMessage() { dupExp.put(Columns.ROW, "1"); dupExp.put(Columns.COLUMN, "1"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, false, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + expResult = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, false, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, expResult.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + expResult); + assertTrue(expResult.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); } @Test @@ -347,9 +379,10 @@ public void importNewEnvNoObsSuccess() { newEnv.put(Columns.ROW, "1"); newEnv.put(Columns.COLUMN, "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); @@ -381,43 +414,53 @@ public void verifyMissingDataThrowsError(boolean commit) { Map noGID = new HashMap<>(base); noGID.remove(Columns.GERMPLASM_GID); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit, newExperimentWorkflowId); Map noExpTitle = new HashMap<>(base); noExpTitle.remove(Columns.EXP_TITLE); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit, newExperimentWorkflowId); Map noExpUnit = new HashMap<>(base); noExpUnit.remove(Columns.EXP_UNIT); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit, newExperimentWorkflowId); Map noExpType = new HashMap<>(base); noExpType.remove(Columns.EXP_TYPE); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit, newExperimentWorkflowId); Map noEnv = new HashMap<>(base); noEnv.remove(Columns.ENV); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit, newExperimentWorkflowId); Map noEnvLoc = new HashMap<>(base); noEnvLoc.remove(Columns.ENV_LOCATION); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); Map noExpUnitId = new HashMap<>(base); noExpUnitId.remove(Columns.EXP_UNIT_ID); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit, newExperimentWorkflowId); Map noExpRep = new HashMap<>(base); noExpRep.remove(Columns.REP_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit, newExperimentWorkflowId); Map noExpBlock = new HashMap<>(base); noExpBlock.remove(Columns.BLOCK_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit, newExperimentWorkflowId); Map noEnvYear = new HashMap<>(base); noEnvYear.remove(Columns.ENV_YEAR); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); } @Test @@ -441,7 +484,8 @@ public void importNewExpWithObsVar() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -491,7 +535,9 @@ public void verifyDiffYearSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); + } @ParameterizedTest @@ -529,7 +575,8 @@ public void verifyDiffLocSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); } @ParameterizedTest @@ -554,7 +601,8 @@ public void importNewExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -594,7 +642,9 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "Red"); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); + } @ParameterizedTest @@ -617,21 +667,24 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + //importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); Map newOU = new HashMap<>(newExp); newOU.put(Columns.EXP_UNIT_ID, "a-2"); newOU.put(Columns.ROW, "1"); newOU.put(Columns.COLUMN, "2"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, true, client, program, mappingId, newExperimentWorkflowId); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); @@ -1022,6 +1075,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { - a new experiment is created after the first experiment - verify the second experiment gets created successfully */ + //TODO: this one @Test @SneakyThrows public void importSecondExpAfterFirstExpWithObs() { @@ -1043,7 +1097,8 @@ public void importSecondExpAfterFirstExpWithObs() { newExpA.put(Columns.COLUMN, "1"); newExpA.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + //JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + JsonObject resultA = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsA = resultA.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsA.size()); @@ -1071,7 +1126,8 @@ public void importSecondExpAfterFirstExpWithObs() { newExpB.put(Columns.COLUMN, "1"); newExpB.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + //JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + JsonObject resultB = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsB = resultB.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsB.size()); @@ -1337,10 +1393,10 @@ private Map assertRowSaved(Map expected, Program assertEquals(expected.get(Columns.GERMPLASM_GID), germplasm.getAccessionNumber()); if(expected.containsKey(Columns.TEST_CHECK) && StringUtils.isNotBlank((String)expected.get(Columns.TEST_CHECK))) { assertEquals(expected.get(Columns.TEST_CHECK), - ou.getObservationUnitPosition() - .getEntryType() - .name() - .substring(0, 1)); + ou.getObservationUnitPosition() + .getEntryType() + .name() + .substring(0, 1)); } assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); @@ -1433,10 +1489,10 @@ private Map assertValidPreviewRow(Map expected, if(traits != null) { assertNotNull(actual.get("observations")); observations = StreamSupport.stream(actual.getAsJsonArray("observations") - .spliterator(), false) - .map(obs -> gson.fromJson(obs.getAsJsonObject() - .getAsJsonObject("brAPIObject"), BrAPIObservation.class)) - .collect(Collectors.toList()); + .spliterator(), false) + .map(obs -> gson.fromJson(obs.getAsJsonObject() + .getAsJsonObject("brAPIObject"), BrAPIObservation.class)) + .collect(Collectors.toList()); ret.put("observations", observations); } @@ -1444,10 +1500,10 @@ private Map assertValidPreviewRow(Map expected, assertEquals(expected.get(Columns.GERMPLASM_GID), germplasm.getAccessionNumber()); if(expected.containsKey(Columns.TEST_CHECK) && StringUtils.isNotBlank((String)expected.get(Columns.TEST_CHECK))) { assertEquals(expected.get(Columns.TEST_CHECK), - ou.getObservationUnitPosition() - .getEntryType() - .name() - .substring(0, 1)); + ou.getObservationUnitPosition() + .getEntryType() + .name() + .substring(0, 1)); } assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); @@ -1518,8 +1574,8 @@ private String yearToSeasonDbId(String year, UUID programId) throws ApiException for (BrAPISeason season : seasons) { if (null == season.getSeasonName() || season.getSeasonName() - .isBlank() || season.getSeasonName() - .equals(year)) { + .isBlank() || season.getSeasonName() + .equals(year)) { return season.getSeasonDbId(); } } @@ -1530,17 +1586,17 @@ private String yearToSeasonDbId(String year, UUID programId) throws ApiException private Program createProgram(String name, String abbv, String key, String referenceSource, List germplasm, List traits) throws ApiException, DoesNotExistException, ValidatorException, BadRequestException { SpeciesEntity validSpecies = speciesDAO.findAll().get(0); SpeciesRequest speciesRequest = SpeciesRequest.builder() - .commonName(validSpecies.getCommonName()) - .id(validSpecies.getId()) - .build(); + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); ProgramRequest programRequest1 = ProgramRequest.builder() - .name(name) - .abbreviation(abbv) - .documentationUrl("localhost:8080") - .objective("To test things") - .species(speciesRequest) - .key(key) - .build(); + .name(name) + .abbreviation(abbv) + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key(key) + .build(); TestUtils.insertAndFetchTestProgram(gson, client, programRequest1); @@ -1609,6 +1665,33 @@ private JsonObject uploadAndVerifyFailure(Program program, File file, String exp JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); + assertEquals(1, rowErrors.size()); + JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); + assertEquals(1, fieldErrors.size()); + JsonObject error = fieldErrors.get(0).getAsJsonObject(); + assertEquals(expectedColumnError, error.get("field").getAsString()); + assertEquals(422, error.get("httpStatusCode").getAsInt()); + + return result; + } + + private JsonObject uploadAndVerifyWorkflowFailure(Program program, File file, String expectedColumnError, boolean commit, String workflowId) throws InterruptedException, IOException { + + //Flowable> call = importTestUtils.uploadDataFile(file, null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(file, null, true, client, program, mappingId, newExperimentWorkflowId); + //JsonObject result = JsonParser.parseString(upload).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); assertEquals(1, rowErrors.size()); JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index 12b79ac15..f5dd37f51 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -97,6 +97,38 @@ public Flowable> uploadDataFile(File file, Map> uploadWorkflowDataFile(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) { + + MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); + + // Upload file + String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", program.getId(), mappingId); + Flowable> call = client.exchange( + POST(uploadUrl, requestBody) + .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + String importId = result.get("importId").getAsString(); + + // Process data + String url = String.format("/programs/%s/import/mappings/%s/workflows/%s/data/%s/%s", program.getId(), mappingId, workflowId, importId, commit ? "commit" : "preview"); + Flowable> processCall = client.exchange( + PUT(url, userData) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + return processCall; + + } + public HttpResponse getUploadedFile(String importId, RxHttpClient client, Program program, String mappingId) throws InterruptedException { Flowable> call = client.exchange( GET(String.format("/programs/%s/import/mappings/%s/data/%s?mapping=true", program.getId(), mappingId, importId)) @@ -123,16 +155,16 @@ public Map setup(RxHttpClient client, Gson gson, DSLContext dsl, // Species Species validSpecies = speciesService.getAll().get(0); SpeciesRequest speciesRequest = SpeciesRequest.builder() - .commonName(validSpecies.getCommonName()) - .id(validSpecies.getId()) - .build(); + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); // Insert program ProgramRequest program = ProgramRequest.builder() - .name("Test Program") - .species(speciesRequest) - .key("TEST") - .build(); + .name("Test Program") + .species(speciesRequest) + .key("TEST") + .build(); Program validProgram = this.insertAndFetchTestProgram(program, client, gson); // Get import @@ -141,18 +173,18 @@ public Map setup(RxHttpClient client, Gson gson, DSLContext dsl, ); HttpResponse response = call.blockingFirst(); String mappingId = JsonParser.parseString(response.body()).getAsJsonObject() - .getAsJsonObject("result") - .getAsJsonArray("data") - .get(0).getAsJsonObject().get("id").getAsString(); + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); BiUserEntity testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), validProgram.getId()); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); return Map.of("program", validProgram, - "mappingId", mappingId, - "testUser", testUser, - "securityFp", securityFp); + "mappingId", mappingId, + "testUser", testUser, + "securityFp", securityFp); } @@ -170,6 +202,43 @@ public JsonObject uploadAndFetch(File file, Map userData, Boolea return result; } + public JsonObject uploadAndFetchWorkflow(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { + Flowable> call = uploadWorkflowDataFile(file, userData, commit, client, program, mappingId, workflowId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + return result; + } + + public JsonObject uploadAndFetchWorkflowNoStatusCheck(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { + Flowable> call = uploadWorkflowDataFile(file, userData, commit, client, program, mappingId, workflowId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + return result; + } + public List createTraits(int numToCreate) { List traits = new ArrayList<>(); for (int i = 0; i < numToCreate; i++) { From 651cffc5dc8fc31e27a0fed557cdf1267ea1e0ee Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:52:11 -0400 Subject: [PATCH 38/50] Removed unused migration --- .../V1.23.0__add_experiment_workflows.sql | 51 ------------------- 1 file changed, 51 deletions(-) delete mode 100644 src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql diff --git a/src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql deleted file mode 100644 index 2c9d4d547..000000000 --- a/src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql +++ /dev/null @@ -1,51 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - Table maps workflows to import mappings and provides required configuration options - - mapping_id - link to importer_mapping that this provides a workflow for - name - name that will be displayed on front end - bean - must match @Named("") annotation on Workflow class - position - for ordering records explicitly, wanted for front end default option and order - */ -CREATE TABLE importer_mapping_workflow -( - like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, - mapping_id UUID NOT NULL, - name TEXT NOT NULL, - bean TEXT NOT NULL, - position INTEGER NOT NULL -); - -ALTER TABLE importer_mapping_workflow - ADD FOREIGN KEY (mapping_id) REFERENCES importer_mapping (id); - -DO -$$ -DECLARE - exp_mapping_id UUID; -BEGIN - exp_mapping_id := (SELECT id FROM importer_mapping WHERE name = 'ExperimentsTemplateMap'); - -INSERT INTO public.importer_mapping_workflow (mapping_id, name, bean, position) -VALUES - (exp_mapping_id, 'Create new experiment', 'CreateNewExperimentWorkflow', 0), - (exp_mapping_id, 'Append experimental dataset', 'AppendOverwritePhenotypesWorkflow', 1), - (exp_mapping_id, 'Create new experimental environment', 'CreateNewEnvironmentWorkflow', 2); -END -$$; \ No newline at end of file From 39fb8cd9ccbddf39f8c759e6654eb7d84364a441 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:03:55 -0400 Subject: [PATCH 39/50] Removed append environment workflow --- .../ExperimentWorkflowNavigator.java | 3 +- .../CreateNewEnvironmentWorkflow.java | 54 ------------------- 2 files changed, 1 insertion(+), 56 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index a498a14b2..9b47c935d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -57,8 +57,7 @@ public List getWorkflows() throws Exception { public enum Workflow { NEW_OBSERVATION("new-experiment","Create new experiment"), - APPEND_OVERWRITE("append-dataset", "Append experimental dataset"), - APPEND_ENVIRONMENT("append-environment", "Create new experimental environment"); + APPEND_OVERWRITE("append-dataset", "Append experimental dataset"); private String id; private String name; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java deleted file mode 100644 index fdb2a9fcc..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; - -import lombok.Getter; -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; - -import javax.inject.Singleton; -import java.util.Optional; - -@Getter -@Singleton -public class CreateNewEnvironmentWorkflow implements ExperimentWorkflow { - private final ExperimentWorkflowNavigator.Workflow workflow; - - public CreateNewEnvironmentWorkflow(){ - this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_ENVIRONMENT; - } - - public Optional process(ImportServiceContext context) { - // Workflow processing the context - ImportWorkflow workflow = ImportWorkflow.builder() - .id(getWorkflow().getId()) - .name(getWorkflow().getName()) - .build(); - - // No-preview result - Optional result = Optional.of(ImportWorkflowResult.builder() - .workflow(workflow) - .importPreviewResponse(Optional.empty()) - .build()); - - // Skip this workflow unless appending a new environment - if (context != null && !this.workflow.isEqual(context.getWorkflow())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return result; - } - - // Start processing the import... - return result; - } - - @Override - public int getOrder() { - return 3; - } - -} From be8ea3844b378a1b4a9e71b9b5797ebe020c57ed Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:36:43 -0400 Subject: [PATCH 40/50] Fix error message --- .../create/workflow/CreateNewExperimentWorkflow.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index b3636a38e..29d7613f1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -101,9 +101,7 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio // Make sure the file does not contain obs unit ids before proceeding if (containsObsUnitIDs(context)) { - // TODO: get file name - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Error detected in file, " + - upload.getUploadFileName() + ". ObsUnitIDs are detected. Import cannot proceed"); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "ObsUnitIDs are detected"); } statusService.updateMessage(upload, "Checking existing experiment objects in brapi service and mapping data"); From d11ef3c63cc8c42c7860147be8a2767b317e96f3 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:42:13 -0400 Subject: [PATCH 41/50] Removed navigator stuff handled in domain import service --- .../experimentObservation/ExperimentImportService.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 013d55012..a9b6c62ac 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.model.imports.DomainImportService; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; @@ -27,14 +26,12 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; -import java.util.List; @Singleton @Slf4j public class ExperimentImportService extends DomainImportService { private final String IMPORT_TYPE_ID = "ExperimentImport"; - private final ExperimentWorkflowNavigator workflowNavigator; // TODO: delete processor fields once WorkflowNavigator is used @Inject @@ -43,7 +40,6 @@ public ExperimentImportService(Provider experimentProcessor ExperimentWorkflowNavigator workflowNavigator) { super(experimentProcessorProvider, processorManagerProvider, workflowNavigator); - this.workflowNavigator = workflowNavigator; } @Override @@ -56,9 +52,5 @@ public String getImportTypeId() { return IMPORT_TYPE_ID; } - @Override - public List getWorkflows() throws Exception{ - return workflowNavigator.getWorkflows(); - } } From eebb4938883656a5f6cb46672d28939cf584d662 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:48:16 -0400 Subject: [PATCH 42/50] Removed some development comments --- .../brapps/importer/model/workflow/ImportContext.java | 2 -- .../brapps/importer/model/workflow/ProcessedData.java | 3 --- 2 files changed, 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java index 53e3a2389..a7b6f7dc3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java @@ -40,8 +40,6 @@ public class ImportContext { private UUID workflowId; private ImportUpload upload; private List importRows; - // TODO: move this out potentially - //private Map mappedBrAPIImport; private Table data; private Program program; private User user; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java index 677f35469..ac3c5ed61 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -28,9 +28,6 @@ @ToString @NoArgsConstructor public class ProcessedData { - // TODO: remove this, already in ImportPreviewResponse - //private Map statistics; - // TODO private Map mappedBrAPIImport; private ImportPreviewResponse importPreviewResponse; } \ No newline at end of file From b5dc3e86ae91d9947c30974f227cb5c3eb2a3175 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 09:49:27 -0400 Subject: [PATCH 43/50] Reference enum for new-experiment --- .../brapps/importer/model/imports/DomainImportService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index d31f0a8d0..9c7a3f261 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.Workflow; @@ -71,7 +72,7 @@ public ImportPreviewResponse process(ImportServiceContext context) // TODO: return results from WorkflowNavigator once processing logic is in separate workflows // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); - if ("new-experiment".equals(context.getWorkflow())) { + if (ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION.getId().equals(context.getWorkflow())) { return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); } else { return processorManagerProvider.get().process(context.getBrAPIImports(), From 2bbf7f3158795c268dcbd2279ed90743f8223978 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:13:42 -0400 Subject: [PATCH 44/50] Change variable name to reflect string type Co-authored-by: mlm483 <128052931+mlm483@users.noreply.github.com> --- .../brapps/importer/model/imports/DomainImportService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 9c7a3f261..3d956743a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -67,8 +67,8 @@ public ImportPreviewResponse process(ImportServiceContext context) throws Exception { Optional.ofNullable(context.getWorkflow()) - .filter(workflow -> !workflow.isEmpty()) - .ifPresent(workflow -> log.info("Workflow: " + workflow)); + .filter(workflowName -> !workflowName.isEmpty()) + .ifPresent(workflowName -> log.info("Workflow: " + workflowName)); // TODO: return results from WorkflowNavigator once processing logic is in separate workflows // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); From ccc141472ccf40930141e2454df089cf55f9be0c Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:20:40 -0400 Subject: [PATCH 45/50] Clarified comment --- .../experiment/create/model/ProcessedPhenotypeData.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java index cd6842476..c81e265cd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java @@ -23,7 +23,8 @@ import java.util.List; import java.util.Map; -// TODO: move to common higher level location +// TODO: move to common higher level location, could be used by both append and create workflows so being located +// in the create namespace won't make sense if we decide to do that in the future. @Getter @Setter @Builder From aa06c6e64c5b651757bbc5b7631d23c1f113b781 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:25:39 -0400 Subject: [PATCH 46/50] Removed old comment --- .../brapps/importer/services/FileImportService.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index da5a68e10..0c2a9309d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -576,11 +576,6 @@ public List getWorkflowsForSystemMapping(UUID mappingId) throws .orElseThrow(() -> new DoesNotExistException("Cannot find mapping config associated with upload.")); BrAPIImportService importService = configManager.getImportServiceById(mappingConfig.getImportTypeId()) .orElseThrow(() -> new DoesNotExistException("Config with that id does not exist")); - // NOTE: - // this is creating a workflow navigator to call getWorkflows - // workflowNavigator.getWorkflows(); - // getWorkflows is creating the ExperimentWorkflow which is injecting the file import service and - // resulting in a circular dependency return importService.getWorkflows(); } From dd6f34862c92bd90758cb8bbad839dac8c484205 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:32:54 -0400 Subject: [PATCH 47/50] Updated doc comment --- .../create/workflow/CreateNewExperimentWorkflow.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 29d7613f1..f007af675 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -139,9 +139,12 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio } /** - * Retrieves the name of the workflow. This is used for logging display purposes. + * Process the import service context and returns an Optional ImportWorkflowResult. * - * @return the name of the workflow + * @param context The import service context to be processed. If null, then it skips processing but returns the result with no-preview. + * @return An Optional ImportWorkflowResult which contains the workflow and import preview response (if available). + * If the context is null, it returns the result with no-preview. + * @throws Exception If any error occurs during the processing. */ public Optional process(ImportServiceContext context) throws Exception { // Workflow processing the context From a6c35902c0bdbd3733a809f72d23a37a627bcfd0 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 15:44:41 -0400 Subject: [PATCH 48/50] Cleaned up ProcessedData code --- .../model/workflow/ProcessedData.java | 1 - .../workflow/CreateNewExperimentWorkflow.java | 3 ++- .../PopulateNewPendingImportObjectsStep.java | 21 ++++++++++--------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java index ac3c5ed61..fc29774f0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -29,5 +29,4 @@ @NoArgsConstructor public class ProcessedData { private Map mappedBrAPIImport; - private ImportPreviewResponse importPreviewResponse; } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index f007af675..d776e4917 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -98,6 +98,7 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio ImportUpload upload = context.getUpload(); boolean commit = context.isCommit(); List importRows = context.getImportRows(); + ProcessedData processedData = new ProcessedData(); // Make sure the file does not contain obs unit ids before proceeding if (containsObsUnitIDs(context)) { @@ -108,7 +109,7 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(context); ProcessContext processContext = populateExistingPendingImportObjectsStep.process(context, phenotypeData); - ProcessedData processedData = populateNewPendingImportObjectsStep.process(processContext, phenotypeData); + populateNewPendingImportObjectsStep.process(processContext, phenotypeData); ValidationErrors validationErrors = validatePendingImportObjectsStep.process(context, processContext.getPendingData(), phenotypeData, processedData); // short circuit if there were validation errors diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index ddb4193ab..f20c8a1a1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -91,18 +91,19 @@ public PopulateNewPendingImportObjectsStep(ExperimentSeasonService experimentSea this.gson = new JSON().getGson(); } - public ProcessedData process(ProcessContext context, ProcessedPhenotypeData phenotypeData) + /** + * TODO: in the future returning ProcessedData rather than modifying in-place would be preferrable. + * + * @param context (modified in-place) + * @param phenotypeData + * @return + * @throws MissingRequiredInfoException + * @throws UnprocessableEntityException + * @throws ApiException + */ + public void process(ProcessContext context, ProcessedPhenotypeData phenotypeData) throws MissingRequiredInfoException, UnprocessableEntityException, ApiException { - - Table data = context.getImportContext().getData(); - ImportUpload upload = context.getImportContext().getUpload(); - ImportContext importContext = context.getImportContext(); - populatePendingImportObjects(context, phenotypeData); - - - // TODO: implement - return new ProcessedData(); } From 1a6de32dc4422765faf51876e4d151898734f542 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 2 Jul 2024 10:18:49 -0400 Subject: [PATCH 49/50] Changed from workflow to workflowId --- .../importer/controllers/UploadController.java | 12 ++++++------ .../importer/model/imports/DomainImportService.java | 8 ++++---- .../importer/model/imports/ImportServiceContext.java | 2 +- .../brapps/importer/services/FileImportService.java | 4 ++-- .../workflow/AppendOverwritePhenotypesWorkflow.java | 2 +- .../create/workflow/CreateNewExperimentWorkflow.java | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java index 5ee6eb062..eb2e2ec50 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java @@ -158,15 +158,15 @@ public HttpResponse> previewData(@PathVariable UUID pro } } - @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflow}/data/{uploadId}/preview") + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/preview") @Produces(MediaType.APPLICATION_JSON) @AddMetadata @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) public HttpResponse> previewData(@PathVariable UUID programId, @PathVariable UUID mappingId, - @PathVariable String workflow, @PathVariable UUID uploadId) { + @PathVariable String workflowId, @PathVariable UUID uploadId) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflow, actingUser, null, false); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, null, false); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { @@ -184,16 +184,16 @@ public HttpResponse> previewData(@PathVariable UUID pro } } - @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflow}/data/{uploadId}/commit") + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/commit") @Produces(MediaType.APPLICATION_JSON) @AddMetadata @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) public HttpResponse> commitData(@PathVariable UUID programId, @PathVariable UUID mappingId, - @PathVariable String workflow, @PathVariable UUID uploadId, + @PathVariable String workflowId, @PathVariable UUID uploadId, @Body @Nullable Map userInput) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflow, actingUser, userInput, true); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, userInput, true); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 3d956743a..4da8efb21 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -66,13 +66,13 @@ public List getWorkflows() throws Exception{ public ImportPreviewResponse process(ImportServiceContext context) throws Exception { - Optional.ofNullable(context.getWorkflow()) - .filter(workflowName -> !workflowName.isEmpty()) - .ifPresent(workflowName -> log.info("Workflow: " + workflowName)); + Optional.ofNullable(context.getWorkflowId()) + .filter(workflowId -> !workflowId.isEmpty()) + .ifPresent(workflowId -> log.info("Workflow: " + workflowId)); // TODO: return results from WorkflowNavigator once processing logic is in separate workflows // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); - if (ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION.getId().equals(context.getWorkflow())) { + if (ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION.getId().equals(context.getWorkflowId())) { return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); } else { return processorManagerProvider.get().process(context.getBrAPIImports(), diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java index 4d052cd33..90e8915f9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -32,7 +32,7 @@ @AllArgsConstructor @NoArgsConstructor public class ImportServiceContext { - private String workflow; + private String workflowId; private List brAPIImports; private Table data; private Program program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 0c2a9309d..e24cfeef2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -420,14 +420,14 @@ public ImportUpload setDynamicColumns(ImportUpload newUpload, Table data, Import return newUpload; } - private void processFile(String workflow, List finalBrAPIImportList, Table data, Program program, + private void processFile(String workflowId, List finalBrAPIImportList, Table data, Program program, ImportUpload upload, User user, Boolean commit, BrAPIImportService importService, AuthenticatedUser actingUser) { // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { ImportServiceContext context = ImportServiceContext.builder() - .workflow(workflow) + .workflowId(workflowId) .brAPIImports(finalBrAPIImportList) .data(data) .program(program) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index 55c34e88a..ea5c388cb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -33,7 +33,7 @@ public Optional process(ImportServiceContext context) { .build()); // Skip this workflow unless appending or overwriting observation data - if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + if (context != null && !this.workflow.isEqual(context.getWorkflowId())) { return Optional.empty(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index d776e4917..ed95014a6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -163,7 +163,7 @@ public Optional process(ImportServiceContext context) thro .build()); // Skip this workflow unless creating a new experiment - if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + if (context != null && !this.workflow.isEqual(context.getWorkflowId())) { return Optional.empty(); } From a488612e506cd767fa7c5acdc69f57b6ad35228b Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:20:07 -0400 Subject: [PATCH 50/50] Change exception handling --- .../model/imports/DomainImportService.java | 10 ++- .../model/workflow/ImportWorkflowResult.java | 1 + .../importer/model/workflow/Workflow.java | 4 +- .../ExperimentWorkflowNavigator.java | 63 ++++++++++--------- .../workflow/CreateNewExperimentWorkflow.java | 27 ++++---- 5 files changed, 60 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 4da8efb21..6cfffe73c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -73,7 +73,15 @@ public ImportPreviewResponse process(ImportServiceContext context) // TODO: return results from WorkflowNavigator once processing logic is in separate workflows // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); if (ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION.getId().equals(context.getWorkflowId())) { - return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); + Optional result = workflowNavigator.process(context); + + // Throw any exceptions caught during workflow processing + if (result.flatMap(ImportWorkflowResult::getCaughtException).isPresent()) { + throw result.flatMap(ImportWorkflowResult::getCaughtException).get(); + } + + return result.flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); + } else { return processorManagerProvider.get().process(context.getBrAPIImports(), List.of(experimentProcessorProvider.get()), diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java index 9304c1ce2..f59b83b55 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java @@ -30,4 +30,5 @@ public class ImportWorkflowResult { private ImportWorkflow workflow; private Optional importPreviewResponse; + private Optional caughtException; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java index dec774e54..17f48ee57 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -9,8 +9,8 @@ @FunctionalInterface public interface Workflow extends Ordered { - Optional process(ImportServiceContext context) throws Exception; - default List getWorkflows() throws Exception { + Optional process(ImportServiceContext context); + default List getWorkflows() { // Default implementation for getWorkflows method return new ArrayList<>(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index 9b47c935d..6acd4aad5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -7,9 +7,9 @@ import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import javax.inject.Singleton; -import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; @Primary @Singleton @@ -20,39 +20,46 @@ public ExperimentWorkflowNavigator(List workflows) { this.workflows = workflows; } + /** + * Process the import service context by executing a series of workflows in order + * + * This method iterates over the list of workflows provided, executing each workflow's process method + * with the given import service context. It then filters out empty results and returns the first non-empty result. + * + * @param context The import service context containing the data to be processed + * @return An Optional containing the first non-empty ImportWorkflowResult from the executed workflows, or an empty Optional if no non-empty result is found + */ @Override - public Optional process(ImportServiceContext context) throws Exception { - - // NOTE: Couldn't throw exception from map lambda - for (ExperimentWorkflow workflow : workflows) { - Optional result = workflow.process(context); - if (result.isPresent()) { - return result; - } - } - - return Optional.empty(); + public Optional process(ImportServiceContext context) { + /** + * Have each workflow in order process the context, returning the first non-empty result + */ + return workflows.stream() + .map(workflow->workflow.process(context)) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst(); } - @Override - public List getWorkflows() throws Exception { - // Each workflow returns in the field workflow the metadata about the workflow that processed the import context. - // Loop over all workflows, processing a null context, to collect just the metadata - - // NOTE: Couldn't throw exception from map lambda - List workflowSummaryList = new ArrayList<>(); - - for (ExperimentWorkflow workflow : workflows) { - Optional result = workflow.process(null); - result.ifPresent(importWorkflowResult -> workflowSummaryList.add(importWorkflowResult.getWorkflow())); - } - - // The order field for each workflow is set to the order in the list + /** + * Retrieves a list of ImportWorkflow objects containing metadata about each workflow that processed the import context. + * + * @return List of ImportWorkflow objects with workflow metadata + */ + public List getWorkflows() { + List workflowSummaryList = workflows.stream() + .map(workflow -> workflow.process(null)) // Process each workflow with a null context + .filter(Optional::isPresent) // Filter out any workflows that do not return a result + .map(Optional::get) // Extract the result from Optional + .map(result -> result.getWorkflow()) // Retrieve the workflow metadata + .collect(Collectors.toList()); // Collect the workflow metadata into a list + + // Set the order field for each workflow based on its position in the list for (int i = 0; i < workflowSummaryList.size(); i++) { - workflowSummaryList.get(i).setOrder(i); + workflowSummaryList.get(i).setOrder(i); // Set the order for each workflow } - return workflowSummaryList; + return workflowSummaryList; // Return the list of workflow metadata } public enum Workflow { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index ed95014a6..d0a4ca975 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -145,9 +145,8 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio * @param context The import service context to be processed. If null, then it skips processing but returns the result with no-preview. * @return An Optional ImportWorkflowResult which contains the workflow and import preview response (if available). * If the context is null, it returns the result with no-preview. - * @throws Exception If any error occurs during the processing. */ - public Optional process(ImportServiceContext context) throws Exception { + public Optional process(ImportServiceContext context) { // Workflow processing the context ImportWorkflow workflow = ImportWorkflow.builder() .id(getWorkflow().getId()) @@ -155,12 +154,11 @@ public Optional process(ImportServiceContext context) thro .build(); // No-preview result - Optional result; - - result = Optional.of(ImportWorkflowResult.builder() + ImportWorkflowResult workflowResult = ImportWorkflowResult.builder() .workflow(workflow) .importPreviewResponse(Optional.empty()) - .build()); + .caughtException(Optional.empty()) + .build(); // Skip this workflow unless creating a new experiment if (context != null && !this.workflow.isEqual(context.getWorkflowId())) { @@ -169,21 +167,22 @@ public Optional process(ImportServiceContext context) thro // Skip processing if no context, but return no-preview result for this workflow if (context == null) { - return result; + return Optional.of(workflowResult); } // TODO: unify usage of single import context type throughout ImportContext importContext = ImportContext.from(context); // Start processing the import... - ImportPreviewResponse response = runWorkflow(importContext); - - result = Optional.of(ImportWorkflowResult.builder() - .workflow(workflow) - .importPreviewResponse(Optional.of(response)) - .build()); + ImportPreviewResponse response; + try { + response = runWorkflow(importContext); + workflowResult.setImportPreviewResponse(Optional.of(response)); + } catch(Exception e) { + workflowResult.setCaughtException(Optional.of(e)); + } - return result; + return Optional.of(workflowResult); } @Override