diff --git a/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java b/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java index 3c7f1d706..923529505 100644 --- a/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java +++ b/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java @@ -17,6 +17,7 @@ package org.breedinginsight.api.model.v1.response; +import io.micronaut.http.HttpStatus; import lombok.*; import lombok.experimental.Accessors; @@ -46,7 +47,6 @@ public void addError(Integer rowNumber, ValidationError validationError){ newRow.addError(validationError); rowErrors.add(newRow); } - public void merge(ValidationErrors validationErrors){ for (RowValidationErrors rowValidationErrors: validationErrors.getRowErrors()){ for (ValidationError validationError: rowValidationErrors.getErrors()) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index e0a61117a..5c9ac0ca1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -37,6 +37,7 @@ import org.breedinginsight.api.model.v1.response.metadata.StatusCode; import org.breedinginsight.api.v1.controller.metadata.AddMetadata; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; +import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; import org.breedinginsight.brapps.importer.services.ImportConfigManager; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; import org.breedinginsight.brapps.importer.services.FileImportService; @@ -208,4 +209,21 @@ public HttpResponse>> getSystemMappings(@Nu Response> response = new Response(metadata, new DataResponse<>(result)); return HttpResponse.ok(response); } + + @Get("/import/mappings/{mappingId}/workflows") + @Produces(MediaType.APPLICATION_JSON) + @AddMetadata + @Secured(SecurityRule.IS_ANONYMOUS) + public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { + + List workflows = fileImportService.getWorkflowsForSystemMapping(mappingId); + + List metadataStatus = new ArrayList<>(); + metadataStatus.add(new Status(StatusCode.INFO, "Successful Query")); + Pagination pagination = new Pagination(workflows.size(), workflows.size(), 1, 0); + Metadata metadata = new Metadata(pagination, metadataStatus); + + Response> response = new Response(metadata, new DataResponse<>(workflows)); + return HttpResponse.ok(response); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java index f9d55bd20..053f1dc3c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java @@ -114,7 +114,7 @@ public HttpResponse> commitData(@PathVariable UUID prog @PathVariable UUID uploadId, @Body @Nullable Map userInput) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, actingUser, userInput, true); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, null, actingUser, userInput, true); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { @@ -140,7 +140,60 @@ public HttpResponse> previewData(@PathVariable UUID pro @PathVariable UUID uploadId) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, actingUser, null, false); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, null, actingUser, null, false); + Response response = new Response(result); + return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + return HttpResponse.notFound(); + } catch (AuthorizationException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.FORBIDDEN, e.getMessage()); + } catch (UnprocessableEntityException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } catch (HttpStatusException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(e.getStatus(), e.getMessage()); + } + } + + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/preview") + @Produces(MediaType.APPLICATION_JSON) + @AddMetadata + @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) + public HttpResponse> previewData(@PathVariable UUID programId, @PathVariable UUID mappingId, + @PathVariable UUID workflowId, @PathVariable UUID uploadId) { + try { + AuthenticatedUser actingUser = securityService.getUser(); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, null, false); + Response response = new Response(result); + return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + return HttpResponse.notFound(); + } catch (AuthorizationException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.FORBIDDEN, e.getMessage()); + } catch (UnprocessableEntityException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } catch (HttpStatusException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(e.getStatus(), e.getMessage()); + } + } + + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/commit") + @Produces(MediaType.APPLICATION_JSON) + @AddMetadata + @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) + public HttpResponse> commitData(@PathVariable UUID programId, @PathVariable UUID mappingId, + @PathVariable UUID workflowId, @PathVariable UUID uploadId, + @Body @Nullable Map userInput) { + try { + AuthenticatedUser actingUser = securityService.getUser(); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, userInput, true); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java new file mode 100644 index 000000000..1b60bda45 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java @@ -0,0 +1,68 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.daos; + +import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; +import org.breedinginsight.dao.db.tables.daos.ImporterMappingWorkflowDao; +import org.jooq.Configuration; +import org.jooq.DSLContext; + +import javax.inject.Inject; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.breedinginsight.dao.db.Tables.*; + +public class ImportMappingWorkflowDAO extends ImporterMappingWorkflowDao { + + private DSLContext dsl; + + @Inject + public ImportMappingWorkflowDAO(Configuration config, DSLContext dsl) { + super(config); + this.dsl = dsl; + } + + /** + * Retrieves a list of ImportMappingWorkflow objects associated with the given mappingId. They are ordered by + * position for proper ordering on the front end. + * + * @param mappingId The UUID of the mapping to retrieve the workflows for. + * @return A list of ImportMappingWorkflow objects. + */ + public List getWorkflowsByImportMappingId(UUID mappingId) { + return dsl.select() + .from(IMPORTER_MAPPING_WORKFLOW) + .where(IMPORTER_MAPPING_WORKFLOW.MAPPING_ID.eq(mappingId)) + .orderBy(IMPORTER_MAPPING_WORKFLOW.POSITION.asc()) + .fetch(ImportMappingWorkflow::parseSQLRecord); + } + + /** + * Retrieves a workflow by its ID. + * + * @param workflowId The ID of the workflow to retrieve. + * @return An Optional containing the ImportMappingWorkflow if found, otherwise an empty Optional. + */ + public Optional getWorkflowById(UUID workflowId) { + return Optional.ofNullable(fetchOneById(workflowId)) + .map(ImportMappingWorkflow::new); + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index 1d520371c..d06454c30 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -17,18 +17,7 @@ package org.breedinginsight.brapps.importer.model.imports; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.ValidatorException; -import tech.tablesaw.api.Table; - -import java.util.List; public interface BrAPIImportService { String getImportTypeId(); @@ -48,6 +37,6 @@ default String getMissingUserInputMsg(String fieldName) { default String getWrongUserInputDataTypeMsg(String fieldName, String typeName) { return String.format("User input, \"%s\" must be an %s", fieldName, typeName); } - ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + ImportPreviewResponse process(ImportServiceContext context) throws Exception; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java new file mode 100644 index 000000000..45393f67c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -0,0 +1,42 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.imports; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import tech.tablesaw.api.Table; +import java.util.List; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ImportServiceContext { + private Workflow workflow; + private List brAPIImports; + private Table data; + private Program program; + private ImportUpload upload; + private User user; + private boolean commit; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index cd795564a..c6f68b251 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -21,6 +21,7 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; @@ -66,12 +67,24 @@ public String getMissingColumnMsg(String columnName) { } @Override - public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + public ImportPreviewResponse process(ImportServiceContext context) throws Exception { ImportPreviewResponse response = null; List processors = List.of(experimentProcessorProvider.get()); - response = processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); + + if (context.getWorkflow() != null) { + log.info("Workflow: " + context.getWorkflow().getName()); + } + + // TODO: change to calling workflow process instead of processor manager + response = processorManagerProvider.get().process(context.getBrAPIImports(), + processors, + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); return response; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index b4eac6b96..ce52f483f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -21,6 +21,7 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.services.processors.GermplasmProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; @@ -62,12 +63,18 @@ public String getImportTypeId() { } @Override - public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + public ImportPreviewResponse process(ImportServiceContext context) throws Exception { ImportPreviewResponse response = null; List processors = List.of(germplasmProcessorProvider.get()); - response = processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); + response = processorManagerProvider.get().process(context.getBrAPIImports(), + processors, + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); return response; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java index 434626e68..2c8e8b7b5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java @@ -21,6 +21,7 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; @@ -59,13 +60,14 @@ public BrAPIImport getImportClass() { } @Override - public ImportPreviewResponse process(List brAPIImports, - Table data, - Program program, - ImportUpload upload, - User user, - Boolean commit) throws Exception { + public ImportPreviewResponse process(ImportServiceContext context) throws Exception { List processors = List.of(sampleProcessorProvider.get()); - return processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); + return processorManagerProvider.get().process(context.getBrAPIImports(), + processors, + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java new file mode 100644 index 000000000..121ef6b19 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java @@ -0,0 +1,47 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import tech.tablesaw.api.Table; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ImportContext { + private UUID workflowId; + private ImportUpload upload; + private List importRows; + private Map mappedBrAPIImport; + private Table data; + private Program program; + private User user; + private boolean commit; +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java new file mode 100644 index 000000000..8dc5800bc --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java @@ -0,0 +1,56 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; +import lombok.experimental.SuperBuilder; + +import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; +import org.jooq.Record; + +import static org.breedinginsight.dao.db.tables.ImporterMappingWorkflowTable.IMPORTER_MAPPING_WORKFLOW; + +@Getter +@Setter +@Accessors(chain=true) +@ToString +@NoArgsConstructor +@SuperBuilder() +public class ImportMappingWorkflow extends ImporterMappingWorkflowEntity { + + + public ImportMappingWorkflow(ImporterMappingWorkflowEntity importMappingWorkflowEntity) { + this.setId(importMappingWorkflowEntity.getId()); + this.setName(importMappingWorkflowEntity.getName()); + this.setBean(importMappingWorkflowEntity.getBean()); + this.setPosition(importMappingWorkflowEntity.getPosition()); + } + public static ImportMappingWorkflow parseSQLRecord(Record record) { + + return ImportMappingWorkflow.builder() + .id(record.getValue(IMPORTER_MAPPING_WORKFLOW.ID)) + .name(record.getValue(IMPORTER_MAPPING_WORKFLOW.NAME)) + .bean(record.getValue(IMPORTER_MAPPING_WORKFLOW.BEAN)) + .position(record.getValue(IMPORTER_MAPPING_WORKFLOW.POSITION)) + .build(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java new file mode 100644 index 000000000..f9f8196c2 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -0,0 +1,30 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; + +import java.util.Map; + +@Data +@ToString +@NoArgsConstructor +public class ProcessedData { + private Map statistics; +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java new file mode 100644 index 000000000..9ea0e7bf9 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -0,0 +1,36 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +public interface Workflow { + + /** + * Processes the given import context and returns the processed data. + * + * @param context the import context containing the necessary data for processing + * @return the processed data + */ + ProcessedData process(ImportContext context); + + /** + * Retrieves the name of the Workflow for logging display purposes. + * + * @return the name of the Workflow + */ + String getName(); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 05c48601d..ef6c5f884 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.micronaut.http.HttpStatus; +import io.micronaut.http.annotation.PathVariable; import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.multipart.CompletedFileUpload; import io.micronaut.http.server.exceptions.InternalServerException; @@ -32,16 +33,22 @@ import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.daos.ImportMappingProgramDAO; +import org.breedinginsight.brapps.importer.daos.ImportMappingWorkflowDAO; import org.breedinginsight.brapps.importer.model.ImportProgress; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; +import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.workflow.WorkflowFactory; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingProgramEntity; +import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import org.breedinginsight.services.ProgramService; @@ -80,12 +87,14 @@ public class FileImportService { private final ImportDAO importDAO; private final DSLContext dsl; private final ImportMappingProgramDAO importMappingProgramDAO; + private final ImportMappingWorkflowDAO importMappingWorkflowDAO; + private final WorkflowFactory workflowFactory; @Inject FileImportService(ProgramUserService programUserService, ProgramService programService, MimeTypeParser mimeTypeParser, ImportMappingDAO importMappingDAO, ObjectMapper objectMapper, MappingManager mappingManager, ImportConfigManager configManager, ImportDAO importDAO, DSLContext dsl, ImportMappingProgramDAO importMappingProgramDAO, - UserService userService) { + ImportMappingWorkflowDAO importMappingWorkflowDAO, WorkflowFactory workflowFactory, UserService userService) { this.programUserService = programUserService; this.programService = programService; this.mimeTypeParser = mimeTypeParser; @@ -97,6 +106,8 @@ public class FileImportService { this.dsl = dsl; this.importMappingProgramDAO = importMappingProgramDAO; this.userService = userService; + this.importMappingWorkflowDAO = importMappingWorkflowDAO; + this.workflowFactory = workflowFactory; } public List getAllImportTypeConfigs() { @@ -322,7 +333,7 @@ public ImportResponse uploadData(UUID programId, UUID mappingId, AuthenticatedUs return response; } - public ImportResponse updateUpload(UUID programId, UUID uploadId, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws + public ImportResponse updateUpload(UUID programId, UUID uploadId, UUID workflowId, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws DoesNotExistException, UnprocessableEntityException, AuthorizationException { Program program = validateRequest(programId, actingUser); @@ -372,7 +383,7 @@ public ImportResponse updateUpload(UUID programId, UUID uploadId, AuthenticatedU } else { brAPIImportList = mappingManager.map(mappingConfig, data); } - processFile(brAPIImportList, data, program, upload, user, commit, importService, actingUser); + processFile(workflowId, brAPIImportList, data, program, upload, user, commit, importService, actingUser); } catch (UnprocessableEntityException e) { log.error(e.getMessage(), e); ImportProgress progress = upload.getProgress(); @@ -418,13 +429,28 @@ public ImportUpload setDynamicColumns(ImportUpload newUpload, Table data, Import return newUpload; } - private void processFile(List finalBrAPIImportList, Table data, Program program, - ImportUpload upload, User user, Boolean commit, BrAPIImportService importService, - AuthenticatedUser actingUser) { + private void processFile(UUID workflowId, List finalBrAPIImportList, Table data, Program program, + ImportUpload upload, User user, Boolean commit, BrAPIImportService importService, + AuthenticatedUser actingUser) { // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { - importService.process(finalBrAPIImportList, data, program, upload, user, commit); + Workflow workflow = null; + if (workflowId != null) { + Optional optionalWorkflow = workflowFactory.getWorkflow(workflowId); + workflow = optionalWorkflow.orElse(null); + } + + ImportServiceContext context = ImportServiceContext.builder() + .workflow(workflow) + .brAPIImports(finalBrAPIImportList) + .data(data) + .program(program) + .upload(upload) + .user(user) + .commit(commit) + .build(); + importService.process(context); } catch (UnprocessableEntityException e) { log.error(e.getMessage(), e); ImportProgress progress = upload.getProgress(); @@ -559,4 +585,10 @@ public List getSystemMappingByName(String name) { List importMappings = importMappingDAO.getSystemMappingByName(name); return importMappings; } + + public List getWorkflowsForSystemMapping(UUID mappingId) { + return importMappingWorkflowDAO.getWorkflowsByImportMappingId(mappingId); + } + + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentProcessor.java new file mode 100644 index 000000000..8207086fe --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentProcessor.java @@ -0,0 +1,42 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.ExperimentWorkflowFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.Workflow; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.services.exceptions.ValidatorException; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.Map; + +@Slf4j +@Prototype +public class ExperimentProcessor { + + private final ExperimentWorkflowFactory experimentWorkflowFactory; + + @Inject + public ExperimentProcessor(ExperimentWorkflowFactory experimentWorkflowFactory) { + this.experimentWorkflowFactory = experimentWorkflowFactory; + } + + public Map process(ImportContext context) + throws ApiException, ValidatorException, MissingRequiredInfoException, UnprocessableEntityException { + + // determine which workflow to use based on the import context + Workflow workflow = experimentWorkflowFactory.getWorkflow(context); + log.info("Importing experiment data using workflow: " + workflow.getName()); + + ProcessedData output = workflow.process(context); + + + return new HashMap<>(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java new file mode 100644 index 000000000..e321c2f59 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -0,0 +1,103 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment; + +import com.google.gson.JsonObject; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.*; +import java.util.stream.Collectors; + +public class ExperimentUtilities { + + public static final CharSequence COMMA_DELIMITER = ","; + public static final String TIMESTAMP_PREFIX = "TS:"; + + public static List importRowsToExperimentObservations(List importRows) { + return importRows.stream() + .map(trialImport -> (ExperimentObservation) trialImport) + .collect(Collectors.toList()); + } + + public static String createObservationUnitKey(ExperimentObservation importRow) { + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); + } + + public static String createObservationUnitKey(String studyName, String obsUnitName) { + return studyName + obsUnitName; + } + + /** + * Returns the single value from the given map, throwing an UnprocessableEntityException if the map does not contain exactly one entry. + * + * @param map The map from which to retrieve the single value. + * @param message The error message to include in the UnprocessableEntityException if the map does not contain exactly one entry. + * @return The single value from the map. + * @throws UnprocessableEntityException if the map does not contain exactly one entry. + */ + public V getSingleEntryValue(Map map, String message) throws UnprocessableEntityException { + if (map.size() != 1) { + throw new UnprocessableEntityException(message); + } + return map.values().iterator().next(); + } + + public static Optional getSingleEntryValue(Map map) { + Optional value = Optional.empty(); + if (map.size() == 1) { + value = Optional.ofNullable(map.values().iterator().next()); + } + return value; + } + + /* + * this will add the given year to the additionalInfo field of the BrAPIStudy (if it does not already exist) + * */ + public void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study, String year) { + JsonObject additionalInfo = study.getAdditionalInfo(); + if (additionalInfo==null){ + additionalInfo = new JsonObject(); + study.setAdditionalInfo(additionalInfo); + } + if( additionalInfo.get(BrAPIAdditionalInfoFields.ENV_YEAR)==null) { + additionalInfo.addProperty(BrAPIAdditionalInfoFields.ENV_YEAR, year); + } + } + + public static Set collateReferenceOUIds(ExpUnitMiddlewareContext context) { + Set referenceOUIds = new HashSet<>(); + boolean hasNoReferenceUnitIds = true; + boolean hasAllReferenceUnitIds = true; + for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + + // Check if ObsUnitID is blank + if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { + hasAllReferenceUnitIds = false; + } else if (referenceOUIds.contains(importRow.getObsUnitID())) { + + // Throw exception if ObsUnitID is repeated + throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); + } else { + // Add ObsUnitID to referenceOUIds + referenceOUIds.add(importRow.getObsUnitID()); + hasNoReferenceUnitIds = false; + } + } + if (!hasNoReferenceUnitIds && !hasAllReferenceUnitIds) { + + // can't proceed if the import has a mix of ObsUnitId for some but not all rows + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, ExpImportProcessConstants.ErrMessage.MISSING_OBS_UNIT_ID_ERROR); + } + return referenceOUIds; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java new file mode 100644 index 000000000..21df19be6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -0,0 +1,50 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; + +import javax.inject.Named; + +/** + * This class represents a workflow for appending and overwriting phenotypes. The bean name must match the appropriate + * bean column value in the import_mapping_workflow db table + */ + +@Prototype +@Named("AppendOverwritePhenotypesWorkflow") +public class AppendOverwritePhenotypesWorkflow implements Workflow { + @Override + public ProcessedData process(ImportContext context) { + // TODO + return null; + } + + /** + * Retrieves the name of the workflow. This is used for logging display purposes. + * + * @return the name of the workflow + */ + @Override + public String getName() { + return "AppendOverwritePhenotypesWorkflow"; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java new file mode 100644 index 000000000..96878ba62 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -0,0 +1,47 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.GetExistingBrAPIData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.Transaction; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ValidateAllRowsHaveIDs; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.Workflow; + +import javax.inject.Inject; +import javax.inject.Provider; + + +@Prototype +public class AppendOverwritePhenotypesWorkflow implements Workflow { + + ExpUnitMiddleware middleware; + Provider transactionProvider; + Provider validateAllRowsHaveIDsProvider; + Provider getExistingBrAPIDataProvider; + @Inject + public AppendOverwritePhenotypesWorkflow(Provider transactionProvider, + Provider validateAllRowsHaveIDsProvider, + Provider getExistingBrAPIDataProvider) { + + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(transactionProvider.get(), + validateAllRowsHaveIDsProvider.get(), + getExistingBrAPIDataProvider.get()); + } + @Override + public ProcessedData process(ImportContext context) { + ExpUnitMiddlewareContext workflowContext = ExpUnitMiddlewareContext.builder().importContext(context).build(); + this.middleware.process(workflowContext); + + // TODO: implement + return new ProcessedData(); + } + + @Override + public String getName() { + return "AppendOverwritePhenotypesWorkflow"; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java new file mode 100644 index 000000000..fed70bb51 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java @@ -0,0 +1,107 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite; + +import io.micronaut.context.annotation.Property; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class ExpUnitContextService { + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public ExpUnitContextService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + } + public List getReferenceUnits(Set expUnitIds, + Program program) throws ApiException { + // Retrieve reference Observation Units based on IDs + return brAPIObservationUnitDAO.getObservationUnitsById(new ArrayList(expUnitIds), program); + } + + public PendingImportObject constructPIOFromExistingUnit(BrAPIObservationUnit unit) { + final PendingImportObject[] pio = new PendingImportObject[]{null}; + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + // Get external reference for the Observation Unit + Optional unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource); + unitXref.ifPresentOrElse( + xref -> { + pio[0] = new PendingImportObject(ImportObjectState.EXISTING, unit, UUID.fromString(xref.getReferenceId())); + }, + () -> { + + // but throw an error if no unit ID + throw new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID"); + } + ); + return pio[0]; + } + + public Map> mapPendingUnitById(List> pios) { + Map> pendingUnitById = new HashMap<>(); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + for (PendingImportObject pio : pios) { + + // Get external reference for the Observation Unit + Optional xref = Utilities.getExternalReference(pio.getBrAPIObject().getExternalReferences(), deltaBreedOUSource); + pendingUnitById.put(xref.get().getReferenceId(),pio); + } + + return pendingUnitById; + } + + public Map> mapPendingUnitByNameNoScope(List> pios, + Program program) { + Map> pendingUnitByNameNoScope = new HashMap<>(); + + for (PendingImportObject pio : pios) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pio.getBrAPIObject().getStudyName(), + program.getKey() + ); + String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pio.getBrAPIObject().getObservationUnitName(), + program.getKey() + ); + pendingUnitByNameNoScope.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName), pio); + } + + return pendingUnitByNameNoScope; + } + + + public List collectMissingOUIds(Set referenceIds, List existingUnits) { + List missingIds = new ArrayList<>(referenceIds); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + Set fetchedIds = existingUnits.stream() + .filter(unit ->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).isPresent()) + .map(unit->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).get().getReferenceId()) + .collect(Collectors.toSet()); + missingIds.removeAll(fetchedIds); + + return missingIds; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java new file mode 100644 index 000000000..8a6bbcb4f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java @@ -0,0 +1,16 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; + +import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.Middleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; + +public abstract class ExpUnitMiddleware extends Middleware { + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // tag an error if it occurred in this local transaction + error.tag(this.getClass().getName()); + + // undo the prior local transaction + return compensatePrior(context, error); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java new file mode 100644 index 000000000..9d85306d4 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java @@ -0,0 +1,117 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class GetExistingBrAPIData extends ExpUnitMiddleware { + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public GetExistingBrAPIData(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + + return processNext(context); + } + + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // tag an error if it occurred in this local transaction + error.tag(this.getClass().getName()); + + // handle the error in the prior local transaction + return compensatePrior(context, error); + } + private Map> fetchReferenceObservationUnits( + ExpUnitMiddlewareContext context) { + Map> pendingUnitById = new HashMap<>(); + try { + // Retrieve reference Observation Units based on IDs + List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( + new ArrayList(context.getExpUnitContext().getReferenceOUIds()), + context.getImportContext().getProgram() + ); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + if (referenceObsUnits.size() == context.getExpUnitContext().getReferenceOUIds().size()) { + + // Iterate through reference Observation Units + for (BrAPIObservationUnit unit : referenceObsUnits) {// Get external reference for the Observation Unit + Optional unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource); + unitXref.ifPresentOrElse( + xref -> { + + // Set pending Observation Unit by its ID + pendingUnitById.put( + xref.getReferenceId(), + new PendingImportObject<>( + ImportObjectState.EXISTING, unit, UUID.fromString(xref.getReferenceId())) + ); + }, + () -> { + + // but throw an error if no unit ID + this.compensate(context, new MiddlewareError(() -> { + throw new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID"); + })); + } + ); + + + } + } else { + // Handle case of missing Observation Units in data store + List missingIds = new ArrayList<>(context.getExpUnitContext().getReferenceOUIds()); + Set fetchedIds = referenceObsUnits.stream().map(unit -> + Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) + .orElseThrow(() -> new InternalServerException("External reference does not exist for Deltabreed ObservationUnit ID")) + .getReferenceId()) + .collect(Collectors.toSet()); + missingIds.removeAll(fetchedIds); + + // throw error reporting any reference IDs with no corresponding stored unit in the brapi data store + this.compensate(context, new MiddlewareError(() -> { + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingIds)); + })); + } + + return pendingUnitById; + } catch (ApiException e) { + + // throw an error if problem getting data from the brapi data store + this.compensate(context, new MiddlewareError(() -> { + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); + try { + throw new ApiException(e); + } catch (ApiException ex) { + throw new RuntimeException(ex); + } + })); + } + return pendingUnitById; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java new file mode 100644 index 000000000..5e3fd28e9 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java @@ -0,0 +1,22 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; + +@Slf4j +public class Transaction extends ExpUnitMiddleware { + // TODO: add member for ExpUnitContext + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + return processNext(context); + } + + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // TODO: handle any error here + + return true; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java new file mode 100644 index 000000000..4a90c4bec --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java @@ -0,0 +1,32 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; + +import java.util.HashSet; +import java.util.Set; + +@Slf4j +public class ValidateAllRowsHaveIDs extends ExpUnitMiddleware { + @Override + public boolean process(ExpUnitMiddlewareContext context) { + + context.getExpUnitContext().setReferenceOUIds(ExperimentUtilities.collateReferenceOUIds(context)); + return processNext(context); + } + + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // tag an error if it occurred in this local transaction + error.tag(this.getClass().getName()); + + // undo the prior local transaction + return compensatePrior(context, error); + } + + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/CreateBrAPITrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/CreateBrAPITrials.java new file mode 100644 index 000000000..d6ea6360b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/CreateBrAPITrials.java @@ -0,0 +1,4 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +public class CreateBrAPITrials { +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java new file mode 100644 index 000000000..aeb88cfe8 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java @@ -0,0 +1,87 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; + +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; + +@Prototype +public class AppendStatistic { + private final HashSet environmentNames; + private final HashSet observationUnitIds; + private final HashSet gids; + private int newCount; + private int existingCount; + private int mutatedCount; + + public AppendStatistic() { + this.environmentNames = new HashSet<>(); + this.observationUnitIds = new HashSet<>(); + this.gids = new HashSet<>(); + this.newCount = 0; + this.existingCount = 0; + this.mutatedCount = 0; + } + + public int incrementNewCount(Integer value) { + int increment = 0; + if (value == null) { + increment = 1; + } else if (value >= 0) { + increment = value; + } + this.newCount += increment; + + return this.newCount; + } + public int incrementExistingCount(Integer value) { + int increment = 0; + if (value == null) { + increment = 1; + } else if (value >= 0) { + increment = value; + } + this.existingCount += increment; + + return this.existingCount; + } + public int incrementMutatedCount(Integer value) { + int increment = 0; + if (value == null) { + increment = 1; + } else if (value >= 0) { + increment = value; + } + this.mutatedCount += increment; + + return this.mutatedCount; + } + public void addEnvironmentName(String name) { + Optional.ofNullable(name).ifPresent(environmentNames::add); + } + public void addObservationUnitId(String id) { + Optional.ofNullable(id).ifPresent(observationUnitIds::add); + } + public void addGid(String gid) { + Optional.ofNullable(gid).ifPresent(gids::add); + } + public Map constructPreviewMap() { + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder().newObjectCount(environmentNames.size()).build(); + ImportPreviewStatistics observationUnitsStats = ImportPreviewStatistics.builder().newObjectCount(observationUnitIds.size()).build(); + ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder().newObjectCount(gids.size()).build(); + ImportPreviewStatistics newStats = ImportPreviewStatistics.builder().newObjectCount(newCount).build(); + ImportPreviewStatistics existingStats = ImportPreviewStatistics.builder().newObjectCount(existingCount).build(); + ImportPreviewStatistics mutatedStats = ImportPreviewStatistics.builder().newObjectCount(mutatedCount).build(); + + return Map.of( + "Environments", environmentStats, + "Observation_Units", observationUnitsStats, + "GIDs", gidStats, + "Observations", newStats, + "Existing_Observations", existingStats, + "Mutated_Observations", mutatedStats + ); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java new file mode 100644 index 000000000..d28a59e2b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java @@ -0,0 +1,39 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.model.Program; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class FieldValidation extends ExpUnitMiddleware { + ObservationUnitService observationUnitService; + + @Inject + public FieldValidation(ObservationUnitService observationUnitService) { + this.observationUnitService = observationUnitService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + + try { + + } catch (Exception e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java new file mode 100644 index 000000000..8e5a10db7 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java @@ -0,0 +1,31 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi.NewPendingBrAPIObjects; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +import javax.inject.Inject; +import javax.inject.Provider; + +@Slf4j +public class ImportPreviewStatistics extends ExpUnitMiddleware { + ExpUnitMiddleware middleware; + private Provider newPendingBrAPIObjectsProvider; + private Provider fieldValidationProvider; + + @Inject + public ImportPreviewStatistics(Provider newPendingBrAPIObjectsProvider, + Provider fieldValidationProvider) { + + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( + newPendingBrAPIObjectsProvider.get(), // Construct Pending import objects for new BrAPI data + fieldValidationProvider.get()); // Validate fields + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + log.debug("generating import preview statistics"); + return this.middleware.process(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java new file mode 100644 index 000000000..48dd9924f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java @@ -0,0 +1,98 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import com.google.gson.Gson; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.model.User; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.BRAPI_REFERENCE_SOURCE; + +public class InitialData extends VisitedObservationData { + boolean isCommit; + String cellData; + String phenoColumnName; + Trait trait; + ExperimentObservation row; + UUID trialId; + UUID studyId; + String unitId; + String studyYear; + BrAPIObservationUnit observationUnit; + User user; + Program program; + @Inject + FieldValidator fieldValidator; + @Inject + StudyService studyService; + @Inject + Gson gson; + + public InitialData(boolean isCommit, + String cellData, + String phenoColumnName, + Trait trait, + ExperimentObservation row, + UUID trialId, + UUID studyId, + String unitId, + String studyYear, + BrAPIObservationUnit observationUnit, User user, + Program program) { + this.isCommit = isCommit; + this.cellData = cellData; + this.phenoColumnName = phenoColumnName; + this.trait = trait; + this.row = row; + this.trialId = trialId; + this.studyId = studyId; + this.unitId = unitId; + this.studyYear = studyYear; + this.observationUnit = observationUnit; + this.user = user; + this.program = program; + } + @Override + public Optional> getValidationErrors() { + List errors = new ArrayList<>(); + + // Validate observation value + fieldValidator.validateField(phenoColumnName, cellData, trait).ifPresent(errors::add); + + return Optional.ofNullable(errors.isEmpty() ? null : errors); + } + + @Override + public PendingImportObject constructPendingObservation() { + String seasonDbId = studyService.seasonDbIdToYear(studyYear, program.getId()); + + // Generate a new ID for the observation + UUID observationID = UUID.randomUUID(); + + // Construct the new observation + BrAPIObservation newObservation = row.constructBrAPIObservation(cellData, phenoColumnName, seasonDbId, observationUnit, isCommit, program, user, BRAPI_REFERENCE_SOURCE, trialId, studyId, UUID.fromString(unitId), observationID); + + // Construct a pending observation with a status set to NEW + return new PendingImportObject<>(ImportObjectState.NEW, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); + + } + + @Override + public void updateTally(AppendStatistic statistic) { + statistic.incrementNewCount(1); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java new file mode 100644 index 000000000..a6d7c23ee --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java @@ -0,0 +1,194 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import io.micronaut.http.HttpStatus; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.FieldValidator; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +public class OverwrittenData extends VisitedObservationData { + @Inject + FieldValidator fieldValidator; + @Inject + Gson gson; + boolean canOverwrite; + boolean isCommit; + String unitId; + Trait trait; + String phenoColumnName; + String timestampColumnName; + String cellData; + String timestamp; + String reason; + BrAPIObservation observation; + UUID userId; + Program program; + + public OverwrittenData(boolean canOverwrite, + boolean isCommit, + String unitId, + Trait trait, + String phenoColumnName, + String timestampColumnName, + String cellData, + String timestamp, + String reason, + BrAPIObservation observation, + UUID userId, + Program program) { + this.canOverwrite = canOverwrite; + this.isCommit = isCommit; + this.unitId = unitId; + this.trait = trait; + this.phenoColumnName = phenoColumnName; + this.timestampColumnName = timestampColumnName; + this.cellData = cellData; + this.timestamp = timestamp; + this.reason = reason; + this.observation = observation; + this.userId = userId; + this.program = program; + } + + @Override + public Optional> getValidationErrors() { + List errors = new ArrayList<>(); + + // Errors for trying to change protected data + if (!canOverwrite) { + if (!isValueMatched()) { + errors.add(new ValidationError(phenoColumnName, String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", unitId, phenoColumnName), HttpStatus.UNPROCESSABLE_ENTITY)); + } + if (!isTimestampMatched()) { + errors.add(new ValidationError(timestampColumnName, String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", unitId, timestampColumnName), HttpStatus.UNPROCESSABLE_ENTITY)); + } + } + + // Validate observation value + fieldValidator.validateField(phenoColumnName, cellData, trait).ifPresent(errors::add); + + // Validate timestamp + fieldValidator.validateField(timestampColumnName, timestamp, null).ifPresent(errors::add); + + return Optional.ofNullable(errors.isEmpty() ? null : errors); + } + + @Override + public PendingImportObject constructPendingObservation() { + // Construct a pending observation with a status set to MUTATED + PendingImportObject pendingUpdatedObservation = new PendingImportObject<>(ImportObjectState.MUTATED, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); + BrAPIObservation update = pendingUpdatedObservation.getBrAPIObject(); + String original = null; + + if (!isValueMatched()) { + // Update the observation value + update.setValue(cellData); + + // Record original observation value for changelog entry + original = observation.getValue(); + } + + if (!isTimestampMatched()) { + // Update the timestamp + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timestamp)); + update.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + + // Add original timestamp to changelog entry + original = Optional.ofNullable(original).map(o -> o + " " + observation.getObservationTimeStamp()).orElse(String.valueOf(observation.getObservationTimeStamp())); + } + + // If the change is to be committed, attach a record of the change as BrAPI observation additional info + if (isCommit) { + // Create the changelog field in observation additional info if it does not already exist + createAdditionalInfoChangeLog(update); + + // Construct a changelog entry + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String rightNow = formatter.format(OffsetDateTime.now()); + ChangeLogEntry entry = new ChangeLogEntry(original, Optional.ofNullable(reason).orElse(""), userId, rightNow); + + // Add the entry to the changelog + update.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(entry).getAsJsonObject()); + } + + return pendingUpdatedObservation; + } + + @Override + public void updateTally(AppendStatistic statistic) { + statistic.incrementMutatedCount(1); + } + + private void createAdditionalInfoChangeLog(BrAPIObservation update) { + if (update.getAdditionalInfo().isJsonNull()) { + update.setAdditionalInfo(new JsonObject()); + update.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + if (update.getAdditionalInfo() != null && !update.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { + update.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + } + + private boolean isValueMatched() { + return !cellData.equals(observation.getValue()); + } + + private boolean isTimestampMatched() { + if (timestamp == null) { + return observation.getObservationTimeStamp() == null; + } else { + return !OffsetDateTime.parse(timestamp).equals(observation.getObservationTimeStamp()); + } + } +// private void validateTimeStampValue(String value, +// String columnHeader, ValidationErrors validationErrors, int row) { +// if (StringUtils.isBlank(value)) { +// log.debug(String.format("skipping validation of observation timestamp because there is no value.\n\tvariable: %s\n\trow: %d", columnHeader, row)); +// return; +// } +// if (!validDateValue(value) && !validDateTimeValue(value)) { +// addRowError(columnHeader, "Incorrect datetime format detected. Expected YYYY-MM-DD or YYYY-MM-DDThh:mm:ss+hh:mm", validationErrors, row); +// } +// +// } + + private boolean validDateValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } + + private boolean validDateTimeValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java new file mode 100644 index 000000000..6c3245966 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java @@ -0,0 +1,38 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import java.util.List; +import java.util.Optional; + +public class UnchangedData extends VisitedObservationData { + BrAPIObservation observation; + Program program; + + public UnchangedData(BrAPIObservation observation, Program program) { + this.observation = observation; + this.program = program; + } + @Override + public Optional> getValidationErrors() { + return Optional.empty(); + } + + @Override + public PendingImportObject constructPendingObservation() { + // Construct a pending observation with a status set to EXISTING + PendingImportObject pendingExistingObservation = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); + + return pendingExistingObservation; + } + + @Override + public void updateTally(AppendStatistic statistic) { + statistic.incrementExistingCount(1); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java new file mode 100644 index 000000000..bdd064b93 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java @@ -0,0 +1,14 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; + +import java.util.List; +import java.util.Optional; + +public abstract class VisitedObservationData { + abstract public Optional> getValidationErrors(); + abstract public PendingImportObject constructPendingObservation(); + abstract public void updateTally(AppendStatistic statistic); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java new file mode 100644 index 000000000..8f7e6362a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java @@ -0,0 +1,49 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi; + +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.jooq.DSLContext; + +import javax.inject.Inject; +import javax.inject.Provider; +import java.math.BigInteger; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.MULTIPLE_EXP_TITLES; + +@Slf4j +public class NewPendingBrAPIObjects extends ExpUnitMiddleware { + ExpUnitMiddleware middleware; + Provider pendingObservationProvider; + @Inject + public NewPendingBrAPIObjects(Provider pendingObservationProvider) { + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(pendingObservationProvider.get()); // Construct new pending observation + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + log.debug("constructing new pending BrAPI objects"); + + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java new file mode 100644 index 000000000..af82901a1 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java @@ -0,0 +1,313 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi; + +import com.google.gson.Gson; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.FileMappingUtil; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.*; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationVariableService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.dao.db.tables.pojos.TraitEntity; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; +import tech.tablesaw.api.Table; +import tech.tablesaw.columns.Column; + +import javax.inject.Inject; +import java.time.OffsetDateTime; +import java.util.*; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.*; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.MULTIPLE_EXP_TITLES; + +@Slf4j +public class PendingObservation extends ExpUnitMiddleware { + StudyService studyService; + ObservationVariableService observationVariableService; + ObservationService observationService; + BrAPIObservationDAO brAPIObservationDAO; + FileMappingUtil fileMappingUtil; + Gson gson; + FieldValidator fieldValidator; + AppendStatistic statistic; + + @Inject + public PendingObservation(StudyService studyService, + ObservationVariableService observationVariableService, + BrAPIObservationDAO brAPIObservationDAO, + ObservationService observationService, + FileMappingUtil fileMappingUtil, + Gson gson, + FieldValidator fieldValidator, + AppendStatistic statistic) { + this.studyService = studyService; + this.observationVariableService = observationVariableService; + this.brAPIObservationDAO = brAPIObservationDAO; + this.observationService = observationService; + this.fileMappingUtil = fileMappingUtil; + this.gson = gson; + this.statistic = statistic; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + log.debug("verifying traits listed in import"); + + // Get all the dynamic columns of the import + ImportUpload upload = context.getImportContext().getUpload(); + Table data = context.getImportContext().getData(); + String[] dynamicColNames = upload.getDynamicColumnNames(); + List> dynamicCols = data.columns(dynamicColNames); + + // Collect the columns for observation variable data + List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); + Set varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toSet()); + + // Collect the columns for observation timestamps + List> timestampCols = dynamicCols.stream().filter(col -> col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); + Set tsNames = timestampCols.stream().map(Column::name).collect(Collectors.toSet()); + + // Construct validation errors for any timestamp columns that don't have a matching variable column + List importRows = context.getImportContext().getImportRows(); + ValidationErrors validationErrors = context.getPendingData().getValidationErrors(); + List tsValErrs = observationVariableService.validateMatchedTimestamps(varNames, timestampCols).orElse(new ArrayList<>()); + for (int i = 0; i < importRows.size(); i++) { + int rowNum = i; + tsValErrs.forEach(validationError -> validationErrors.addError(rowNum, validationError)); + } + + // Stop processing the import if there are unmatched timestamp columns + if (tsValErrs.size() > 0) { + this.compensate(context, new MiddlewareError(() -> { + // any handling... + })); + } + + //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval + Map> tsColByPheno = timestampCols.stream().collect(Collectors.toMap(col -> col.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), col -> col)); + + // Add the map to the context for use in processing import + context.getPendingData().setTimeStampColByPheno(tsColByPheno); + + try { + // Fetch the traits named in the observation variable columns + Program program = context.getImportContext().getProgram(); + List traits = observationVariableService.fetchTraitsByName(varNames, program); + + // Map trait by phenotype column name + Map traitByPhenoColName = traits.stream().collect( + Collectors.toMap( + trait -> trait.getObservationVariableName().toUpperCase(), // Use uppercase keys for case-insensitivity + trait -> trait, + (trait1, trait2) -> trait1, // Merge function + CaseInsensitiveMap::new // Supplier for creating a CaseInsensitiveMap + ) + ); + + // Sort the traits to match the order of the headers in the import file + List sortedTraits = fileMappingUtil.sortByField(List.copyOf(varNames), new ArrayList<>(traits), TraitEntity::getObservationVariableName); + + // Get the pending observation dataset + PendingImportObject pendingTrial = ExperimentUtilities.getSingleEntryValue(context.getPendingData().getTrialByNameNoScope()).orElseThrow(()->new UnprocessableEntityException(MULTIPLE_EXP_TITLES.getValue())); + String datasetName = String.format("Observation Dataset [%s-%s]", program.getKey(), pendingTrial.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString()); + PendingImportObject pendingDataset = context.getPendingData().getObsVarDatasetByName().get(datasetName); + + // Add new phenotypes to the pending observation dataset list (NOTE: "obsVarName [programKey]" is used instead of obsVarDbId) + // TODO: Change to using actual dbIds as per the BrAPI spec, instead of namespaced obsVar names (necessary for Breedbase) + Set datasetObsVarDbIds = pendingDataset.getBrAPIObject().getData().stream().collect(Collectors.toSet()); + Set phenoDbIds = sortedTraits.stream().map(t->Utilities.appendProgramKey(t.getObservationVariableName(), program.getKey())).collect(Collectors.toSet()); + phenoDbIds.removeAll(datasetObsVarDbIds); + pendingDataset.getBrAPIObject().getData().addAll(phenoDbIds); + + // Update pending status + if (ImportObjectState.EXISTING == pendingDataset.getState()) { + pendingDataset.setState(ImportObjectState.MUTATED); + } + + // Read any observation data stored for these traits + log.debug("fetching observation data stored for traits"); + Set ouDbIds = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(u -> u.getBrAPIObject().getObservationUnitDbId()).collect(Collectors.toSet()); + Set varDbIds = sortedTraits.stream().map(t->t.getObservationVariableDbId()).collect(Collectors.toSet()); + List observations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, varDbIds, program); + + // Construct helper lookup tables to use for hashing stored observation data + Map unitNameByDbId = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, BrAPIObservationUnit::getObservationUnitName)); + Map variableNameByDbId = sortedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableDbId, Trait::getObservationVariableName)); + Map studyNameByDbId = context.getPendingData().getStudyByNameNoScope().values().stream() + .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) + .map(PendingImportObject::getBrAPIObject) + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); + + // Hash stored observation data using a signature of unit, variable, and study names + Map observationByObsHash = observations.stream().collect(Collectors.toMap(o->{ + return observationService.getObservationHash(unitNameByDbId.get(o.getObservationUnitDbId()), + variableNameByDbId.get(o.getObservationVariableDbId()), + studyNameByDbId.get(o.getStudyDbId())); + }, o->o)); + + // Add the observation data map to the context for use in processing import + context.getPendingData().setExistingObsByObsHash(observationByObsHash); + + // Build new pending observation data for each phenotype + Map> pendingObservationByHash = new HashMap<>(); + + // Process observation data for each row + for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { + Integer rowNum = i; + ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + + // Construct the pending import for the row + PendingImport mappedImportRow = context.getImportContext().getMappedBrAPIImport().getOrDefault(rowNum, new PendingImport()); + String unitId = row.getObsUnitID(); + mappedImportRow.setTrial(context.getExpUnitContext().getPendingTrialByOUId().get(unitId)); + mappedImportRow.setLocation(context.getExpUnitContext().getPendingLocationByOUId().get(unitId)); + mappedImportRow.setStudy(context.getExpUnitContext().getPendingStudyByOUId().get(unitId)); + mappedImportRow.setObservationUnit(context.getExpUnitContext().getPendingObsUnitByOUId().get(unitId)); + mappedImportRow.setGermplasm(context.getExpUnitContext().getPendingGermplasmByOUId().get(unitId)); + + // For each phenotype, construct the pending observations + for (Column column : phenotypeCols) { + String cellData = column.getString(rowNum); + + // Generate hash for looking up prior observation data + String studyName = context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getStudyName(); + String unitName = context.getExpUnitContext().getPendingObsUnitByOUId().get(unitId).getBrAPIObject().getObservationUnitName(); + String phenoColumnName = column.name(); + String observationHash = observationService.getObservationHash(unitName, phenoColumnName, studyName); + + // Get timestamp if associated column + var cell = new Object() { // mutable reference object to make timestamp accessible in anonymous methods + String timestamp = null; + }; + String tsColumnName = null; + if (tsColByPheno.containsKey(phenoColumnName)) { + cell.timestamp = tsColByPheno.get(phenoColumnName).getString(rowNum); + tsColumnName = tsColByPheno.get(phenoColumnName).name(); + + // If timestamp is not valid, set to midnight + fieldValidator.validateField(tsColumnName, cell.timestamp, null).ifPresent(err->{ + cell.timestamp += MIDNIGHT; + validationErrors.addError(rowNum, err); + }); + + } + + VisitedObservationData processedData = null; + // Is there prior observation data for this unit + var? + if (observationByObsHash.containsKey(observationHash)) { + + // Clone the prior observation + BrAPIObservation observation = gson.fromJson(gson.toJson(observationByObsHash.get(observationHash)), BrAPIObservation.class); + + // Is there a change to the prior data? + if (!cellData.equals(observation.getValue()) || (cell.timestamp != null && !OffsetDateTime.parse(cell.timestamp).equals(observation.getObservationTimeStamp()))) { + + // Is prior data protected? + boolean canOverwrite = context.getImportContext().isCommit() && "false".equals( row.getOverwrite() == null ? "false" : row.getOverwrite()); + + // Clone the trait + Trait changeTrait = gson.fromJson(gson.toJson(traitByPhenoColName.get(phenoColumnName)), Trait.class); + + // Create new instance of OverwrittenData + processedData = new OverwrittenData(canOverwrite, + context.getImportContext().isCommit(), + unitId, + changeTrait, + phenoColumnName, + tsColumnName, + cellData, + cell.timestamp, + Optional.ofNullable(row.getOverwriteReason()).orElse(""), + observation, + context.getImportContext().getUser().getId(), + program); + } else { + + // create new instance of UnchangedData + processedData = new UnchangedData(observation, program); + } + + } else { + + // Clone the observation unit and trait + BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getExpUnitContext().getPendingObsUnitByOUId().get(row.getExpUnitId()).getBrAPIObject()), BrAPIObservationUnit.class); + Trait initialTrait = gson.fromJson(gson.toJson(traitByPhenoColName.get(phenoColumnName)), Trait.class); + + // create new instance of InitialData + processedData = new InitialData(context.getImportContext().isCommit(), + cellData, + phenoColumnName, + initialTrait, + row, + pendingTrial.getId(), + context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getId(), + unitId, + context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getSeasons().get(0), + observationUnit, + context.getImportContext().getUser(), + context.getImportContext().getProgram()); + } + + // Validate processed data + processedData.getValidationErrors().ifPresent(errList -> errList.forEach(e->validationErrors.addError(rowNum, e))); + + // Update import preview statistics + processedData.updateTally(statistic); + statistic.addEnvironmentName(studyName); + // TODO: change null values to actual data + // TODO: change signature to take two args, studyName and unitName + statistic.addObservationUnitId(null); + statistic.addGid(null); + + // Construct a pending observation + PendingImportObject pendingProcessedData = processedData.constructPendingObservation(); + + // Set the new pending observation in the pending import for the row + mappedImportRow.getObservations().add(pendingProcessedData); + + // Add pending observation to map + pendingObservationByHash.put(observationHash, pendingProcessedData); + } + + // Set the pending import for the row + context.getImportContext().getMappedBrAPIImport().put(rowNum, mappedImportRow); + } + + // Add the pending observation map to the context for use in processing the import + context.getPendingData().setPendingObservationByHash(pendingObservationByHash); + } catch (DoesNotExistException | ApiException | UnprocessableEntityException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java new file mode 100644 index 000000000..180e6957a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java @@ -0,0 +1,42 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +import javax.inject.Inject; +import javax.inject.Provider; + +@Slf4j +public class RequiredBrAPIData extends ExpUnitMiddleware { + ExpUnitMiddleware middleware; + Provider requiredObservationUnitsProvider; + Provider requiredTrialsProvider; + Provider requiredStudiesProvider; + Provider requiredLocationsProvider; + Provider requiredDatasetsProvider; + Provider requiredGermplasmProvider; + + @Inject + public RequiredBrAPIData(Provider requiredObservationUnitsProvider, + Provider requiredTrialsProvider, + Provider requiredStudiesProvider, + Provider requiredLocationsProvider, + Provider requiredDatasetsProvider, + Provider requiredGermplasmProvider) { + + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( + requiredObservationUnitsProvider.get(), // Fetch the BrAPI units for the required exp unit ids + requiredTrialsProvider.get(), // Fetch the BrAPI trials belonging to the exp units + requiredStudiesProvider.get(), // Fetch the BrAPI studies belonging to the exp units + requiredLocationsProvider.get(), // Fetch the BrAPI locations belonging to the exp units + requiredDatasetsProvider.get(), // Fetch the dataset belonging to the exp units + requiredGermplasmProvider.get()); // Fetch the germplasm belonging to the exp units + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + log.debug("reading required BrAPI data from BrAPI service"); + return this.middleware.process(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java new file mode 100644 index 000000000..a2536ecf0 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java @@ -0,0 +1,72 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; +import org.breedinginsight.model.Program; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.Map; + +@Slf4j +public class RequiredDatasets extends ExpUnitMiddleware { + private final DatasetService datasetService; + + @Inject + public RequiredDatasets(DatasetService datasetService) { + this.datasetService = datasetService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + String datasetId; + BrAPIListDetails dataset = null; + PendingImportObject pendingDataset; + Map> pendingTrialByNameNoScope; + Map> pendingDatasetByName; + + program = context.getImportContext().getProgram(); + pendingTrialByNameNoScope = context.getPendingData().getTrialByNameNoScope(); + + // nothing to do if there are no trials with dataset ids + if (pendingTrialByNameNoScope.size() == 0 || + !pendingTrialByNameNoScope.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + return processNext(context); + } + log.debug("fetching from BrAPI service, datasets belonging to required units"); + + // Get the id of the dataset belonging to the required exp units + datasetId = pendingTrialByNameNoScope.values().iterator().next().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + try { + // Get the dataset belonging to required exp units + dataset = datasetService.fetchDatasetById(datasetId, program).orElseThrow(ApiException::new); + + // Construct the pending dataset from the BrAPI observation variable list + pendingDataset = datasetService.constructPIOFromDataset(dataset, program); + + // Construct a hashmap to look up the pending dataset by dataset name + pendingDatasetByName = new HashMap<>(); + pendingDatasetByName.put(dataset.getListName(), pendingDataset); + + // Add the map to the context for use in processing import + context.getPendingData().setObsVarDatasetByName(pendingDatasetByName); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java new file mode 100644 index 000000000..1d40ea781 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java @@ -0,0 +1,72 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.GermplasmService; +import org.breedinginsight.model.Program; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class RequiredGermplasm extends ExpUnitMiddleware { + private final GermplasmService germplasmService; + + @Inject + public RequiredGermplasm(GermplasmService germplasmService) { + + this.germplasmService = germplasmService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Set germplasmDbIds; + List brapiGermplasm = null; + List> pendingGermplasm; + Map> pendingGermplasmByGID; + Map> pendingUnitByNameNoScope; + + program = context.getImportContext().getProgram(); + pendingUnitByNameNoScope = context.getPendingData().getObservationUnitByNameNoScope(); + + // nothing to do if there are no observation units + if (pendingUnitByNameNoScope.size() == 0) { + return processNext(context); + } + log.debug("fetching from BrAPI service, germplasm belonging to required units"); + + // Get the dbIds of the germplasm belonging to the required exp units + germplasmDbIds = pendingUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + try { + // Get the dataset belonging to required exp units + brapiGermplasm = germplasmService.fetchGermplasmByDbId(new HashSet<>(germplasmDbIds), program); + + // Construct the pending germplasm from the BrAPI locations + pendingGermplasm = brapiGermplasm.stream().map(germplasmService::constructPIOFromBrapiGermplasm).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending germplasm by gid + pendingGermplasmByGID = pendingGermplasm.stream().collect(Collectors.toMap(germplasmService::getGIDFromGermplasmPIO, pio -> pio)); + + // Add the map to the context for use in processing import + context.getPendingData().setExistingGermplasmByGID(pendingGermplasmByGID); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java new file mode 100644 index 000000000..fed9bf455 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java @@ -0,0 +1,72 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +public class RequiredLocations extends ExpUnitMiddleware { + LocationService locationService; + + @Inject + public RequiredLocations(LocationService locationService) { + this.locationService = locationService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Set locationDbIds; + List brapiLocations; + List> pendingLocations; + Map> pendingStudyByNameNoScope; + Map> pendingLocationByName; + + program = context.getImportContext().getProgram(); + pendingStudyByNameNoScope = context.getPendingData().getStudyByNameNoScope(); + + // nothing to do if there are no required units + if (pendingStudyByNameNoScope.size() == 0) { + return processNext(context); + } + log.debug("fetching from BrAPI service, locations belonging to required units"); + + // Get the dbIds of the studies belonging to the required exp units + locationDbIds = pendingStudyByNameNoScope.values().stream().map(pio -> pio.getBrAPIObject().getLocationDbId()).collect(Collectors.toSet()); + try { + // Get the locations belonging to required exp units + brapiLocations = locationService.fetchLocationsByDbId(locationDbIds, program); + + // Construct the pending locations from the BrAPI locations + pendingLocations = brapiLocations.stream().map(locationService::constructPIOFromBrapiLocation).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending location by location name + pendingLocationByName = pendingLocations.stream().collect(Collectors.toMap(pio -> pio.getBrAPIObject().getName(), pio -> pio)); + + // Add the map to the context for use in processing import + context.getPendingData().setLocationByName(pendingLocationByName); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java new file mode 100644 index 000000000..ba19b96f3 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java @@ -0,0 +1,65 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.model.Program; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class RequiredObservationUnits extends ExpUnitMiddleware { + ObservationUnitService observationUnitService; + + @Inject + public RequiredObservationUnits(ObservationUnitService observationUnitService) { + this.observationUnitService = observationUnitService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Set expUnitIds; + List missingIds; + List brapiUnits; + List> pendingUnits; + Map> pendingUnitById; + Map> pendingUnitByNameNoScope; + + log.debug("fetching required exp units from BrAPI service"); + program = context.getImportContext().getProgram(); + + // Collect deltabreed-generated exp unit ids listed in the import + expUnitIds = context.getExpUnitContext().getReferenceOUIds(); + try { + // For each id fetch the observation unit from the brapi data store + brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); + + // Construct pending import objects from the units + pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending unit by ID + pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); + + // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed + pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); + + // add maps to the context for use in processing import + context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); + context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java new file mode 100644 index 000000000..493ae0021 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java @@ -0,0 +1,73 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +public class RequiredStudies extends ExpUnitMiddleware { + StudyService studyService; + + @Inject + public RequiredStudies(StudyService studyService) { + this.studyService = studyService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Map> pendingUnitByNameNoScope; + Set studyDbIds; + List brAPIStudies; + List> pendingStudies; + Map> pendingStudyByNameNoScope; + + program = context.getImportContext().getProgram(); + pendingUnitByNameNoScope = context.getPendingData().getObservationUnitByNameNoScope(); + + // nothing to do if there are no required units + if (pendingUnitByNameNoScope.size() == 0) { + return processNext(context); + } + log.debug("fetching from BrAPI service studies belonging to required units"); + + // Get the dbIds of the studies belonging to the required exp units + studyDbIds = pendingUnitByNameNoScope.values().stream().map(studyService::getStudyDbIdBelongingToPendingUnit).collect(Collectors.toSet()); + + try { + // Get the BrAPI studies belonging to required exp units + brAPIStudies = studyService.fetchBrapiStudiesByDbId(studyDbIds, program); + + // Construct the pending studies from the BrAPI trials + pendingStudies = brAPIStudies.stream().map(pio -> studyService.constructPIOFromBrapiStudy(pio, program)).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending study by study name with the program key removed + pendingStudyByNameNoScope = pendingStudies.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKeyAndUnknownAdditionalData(pio.getBrAPIObject().getStudyName(), program.getKey()), pio -> pio)); + + // Add the map to the context for use in processing import + context.getPendingData().setStudyByNameNoScope(pendingStudyByNameNoScope); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java new file mode 100644 index 000000000..aac07d77c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java @@ -0,0 +1,70 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class RequiredTrials extends ExpUnitMiddleware { + TrialService trialService; + + @Inject + public RequiredTrials(TrialService trialService) { + this.trialService = trialService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Map> pendingUnitByNameNoScope; + Set trialDbIds; + List brAPITrials; + List> pendingTrials; + Map> pendingTrialByNameNoScope; + + program = context.getImportContext().getProgram(); + pendingUnitByNameNoScope = context.getPendingData().getObservationUnitByNameNoScope(); + + // nothing to do if there are no required units + if (pendingUnitByNameNoScope.size() == 0) { + return processNext(context); + } + log.debug("fetching from BrAPI service, trials belonging to required units"); + + // Get the dbIds of the trials belonging to the required exp units + trialDbIds = pendingUnitByNameNoScope.values().stream().map(pendingUnit -> trialService.getTrialDbIdBelongingToPendingUnit(pendingUnit, program)).collect(Collectors.toSet()); + try { + // Get the BrAPI trials belonging to required exp units + brAPITrials = trialService.fetchBrapiTrialsByDbId(trialDbIds, program); + + // Construct the pending trials from the BrAPI trials + pendingTrials = brAPITrials.stream().map(trialService::constructPIOFromBrapiTrial).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending trial by trial name with the program key removed + pendingTrialByNameNoScope = pendingTrials.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKey(pio.getBrAPIObject().getTrialName(), program.getKey()), pio -> pio)); + + // Add the map to the context for use in processing import + context.getPendingData().setTrialByNameNoScope(pendingTrialByNameNoScope); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/DateValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/DateValidator.java new file mode 100644 index 000000000..615da28ae --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/DateValidator.java @@ -0,0 +1,65 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; + +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.model.Trait; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; +import static org.breedinginsight.dao.db.enums.DataType.DATE; + +@Slf4j +@Singleton +public class DateValidator implements ObservationValidator { + @Inject + ObservationService observationService; + private final String dateMessage = "Incorrect date format detected. Expected YYYY-MM-DD"; + private final String dateTimeMessage = "Incorrect datetime format detected. Expected YYYY-MM-DD or YYYY-MM-DDThh:mm:ss+hh:mm"; + + public DateValidator(ObservationService observationService) { + this.observationService = observationService; + } + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + if (observationService.isBlankObservation(value)) { + log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + if (observationService.isNAObservation(value)) { + log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + // Is this a timestamp field? + if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + if (!observationService.validDateValue(value) || !observationService.validDateTimeValue(value)) { + return Optional.of(new ValidationError(fieldName, dateTimeMessage, HttpStatus.UNPROCESSABLE_ENTITY)); + } + + } else { + + // Skip if there is no trait data + if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { + return Optional.empty(); + } + + // Skip if this is not a date trait + if (!DATE.equals(variable.getScale().getDataType())) { + return Optional.empty(); + } + + // Validate date + if (!observationService.validDateValue(value)) { + return Optional.of(new ValidationError(fieldName, dateMessage, HttpStatus.UNPROCESSABLE_ENTITY)); + } + } + + return Optional.empty(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/FieldValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/FieldValidator.java new file mode 100644 index 000000000..724aa01d3 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/FieldValidator.java @@ -0,0 +1,28 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; + +import io.micronaut.context.annotation.Primary; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.model.Trait; + +import javax.inject.Singleton; +import java.util.List; +import java.util.Optional; + +@Primary +@Singleton +public class FieldValidator implements ObservationValidator { + private final List validators; + + public FieldValidator(List validators) { + this.validators = validators; + } + + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + return validators.stream() + .map(validator->validator.validateField(fieldName, value, variable)) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NominalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NominalValidator.java new file mode 100644 index 000000000..7f2cf50df --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NominalValidator.java @@ -0,0 +1,59 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; + +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.model.Trait; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; +import static org.breedinginsight.dao.db.enums.DataType.NOMINAL; + +@Slf4j +@Singleton +public class NominalValidator implements ObservationValidator { + @Inject + ObservationService observationService; + + public NominalValidator(ObservationService observationService) { + this.observationService = observationService; + } + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + if (observationService.isBlankObservation(value)) { + log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + if (observationService.isNAObservation(value)) { + log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + // Skip if field is a timestamp + if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + return Optional.empty(); + } + + // Skip if there is no trait data + if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { + return Optional.empty(); + } + + // Skip if this is not an ordinal trait + if (!NOMINAL.equals(variable.getScale().getDataType())) { + return Optional.empty(); + } + + // Validate categories + if (!observationService.validCategory(variable.getScale().getCategories(), value)) { + return Optional.of(new ValidationError(fieldName, "Undefined nominal category detected", HttpStatus.UNPROCESSABLE_ENTITY)); + } + + return Optional.empty(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NumericalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NumericalValidator.java new file mode 100644 index 000000000..d90d8d115 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NumericalValidator.java @@ -0,0 +1,66 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; + +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.model.Trait; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.math.BigDecimal; +import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; +import static org.breedinginsight.dao.db.enums.DataType.NUMERICAL; + +@Slf4j +@Singleton +public class NumericalValidator implements ObservationValidator { + @Inject + ObservationService observationService; + + public NumericalValidator(ObservationService observationService) { + this.observationService = observationService; + } + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + if (observationService.isBlankObservation(value)) { + log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + if (observationService.isNAObservation(value)) { + log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + // Skip if field is a timestamp + if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + return Optional.empty(); + } + + // Skip if there is no trait data + if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { + return Optional.empty(); + } + + // Skip if this is not a numerical trait + if (!NUMERICAL.equals(variable.getScale().getDataType())) { + return Optional.empty(); + } + + Optional number = observationService.validNumericValue(value); + Optional validationError = number + .flatMap(num -> { + if (!observationService.validNumericRange(num, variable.getScale())) { + return Optional.of(new ValidationError(fieldName, "Value outside of min/max range detected", HttpStatus.UNPROCESSABLE_ENTITY)); + } + return Optional.empty(); + }) + .or(() -> Optional.of(new ValidationError(fieldName, "Non-numeric text detected", HttpStatus.UNPROCESSABLE_ENTITY))); + + return validationError; + + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/ObservationValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/ObservationValidator.java new file mode 100644 index 000000000..e378cd228 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/ObservationValidator.java @@ -0,0 +1,13 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; + +import io.micronaut.core.annotation.NonNull; +import io.micronaut.core.order.Ordered; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.model.Trait; + +import java.util.Optional; + +@FunctionalInterface +public interface ObservationValidator extends Ordered { + Optional validateField(@NonNull String fieldName, @NonNull String value, Trait variable); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/OrdinalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/OrdinalValidator.java new file mode 100644 index 000000000..76fbcab04 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/OrdinalValidator.java @@ -0,0 +1,59 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; + +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.model.Trait; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; +import static org.breedinginsight.dao.db.enums.DataType.ORDINAL; + +@Slf4j +@Singleton +public class OrdinalValidator implements ObservationValidator { + @Inject + ObservationService observationService; + + public OrdinalValidator(ObservationService observationService) { + this.observationService = observationService; + } + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + if (observationService.isBlankObservation(value)) { + log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + if (observationService.isNAObservation(value)) { + log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + // Skip if field is a timestamp + if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + return Optional.empty(); + } + + // Skip if there is no trait data + if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { + return Optional.empty(); + } + + // Skip if this is not an ordinal trait + if (!ORDINAL.equals(variable.getScale().getDataType())) { + return Optional.empty(); + } + + // Validate categories + if (!observationService.validCategory(variable.getScale().getCategories(), value)) { + return Optional.of(new ValidationError(fieldName, "Undefined ordinal category detected", HttpStatus.UNPROCESSABLE_ENTITY)); + } + + return Optional.empty(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java new file mode 100644 index 000000000..6988cbf89 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java @@ -0,0 +1,31 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; + +import lombok.Getter; +import lombok.Setter; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.ProgramLocation; +import tech.tablesaw.columns.Column; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@Getter +@Setter +public class ExpUnitContext { + private Set referenceOUIds = new HashSet<>(); + private Map> pendingTrialByOUId = new HashMap<>(); + private Map> pendingStudyByOUId = new HashMap<>(); + private Map> pendingObsUnitByOUId = new HashMap<>(); + private Map> pendingObsDatasetByOUId = new HashMap<>(); + private Map> pendingLocationByOUId = new HashMap<>(); + private Map> pendingGermplasmByOUId = new HashMap<>(); + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java new file mode 100644 index 000000000..28eb14e29 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -0,0 +1,35 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; + +import lombok.*; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.ProgramLocation; +import tech.tablesaw.columns.Column; + +import java.util.HashMap; +import java.util.Map; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class PendingData { + private Map> observationUnitByNameNoScope; + private Map> trialByNameNoScope; + private Map> studyByNameNoScope; + private Map> locationByName; + private Map> obsVarDatasetByName; + private Map> existingGermplasmByGID; + private Map> pendingObservationByHash; + private Map> timeStampColByPheno; + private Map existingObsByObsHash; + private ValidationErrors validationErrors; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java new file mode 100644 index 000000000..5cd943645 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -0,0 +1,464 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.steps; + +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.*; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.*; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.ProgramLocationService; +import org.breedinginsight.utilities.Utilities; + + + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Prototype +@Slf4j +public class GetExistingProcessingStep implements ProcessingStep { + + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private final BrAPITrialDAO brAPITrialDAO; + private final BrAPIStudyDAO brAPIStudyDAO; + private final ProgramLocationService locationService; + private final BrAPIListDAO brAPIListDAO; + private final BrAPIGermplasmDAO brAPIGermplasmDAO; + private final StudyService studyService; + private final TrialService trialService; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public GetExistingProcessingStep(BrAPIObservationUnitDAO brAPIObservationUnitDAO, + BrAPITrialDAO brAPITrialDAO, + BrAPIStudyDAO brAPIStudyDAO, + ProgramLocationService locationService, + BrAPIListDAO brAPIListDAO, + BrAPIGermplasmDAO brAPIGermplasmDAO, + StudyService studyService, + TrialService trialService) { + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + this.brAPITrialDAO = brAPITrialDAO; + this.brAPIStudyDAO = brAPIStudyDAO; + this.locationService = locationService; + this.brAPIListDAO = brAPIListDAO; + this.brAPIGermplasmDAO = brAPIGermplasmDAO; + this.studyService = studyService; + this.trialService = trialService; + } + + @Override + public PendingData process(ImportContext input) { + + List experimentImportRows = ExperimentUtilities.importRowsToExperimentObservations(input.getImportRows()); + Program program = input.getProgram(); + + // Populate pending objects with existing status + Map> observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); + Map> trialByNameNoScope = initializeTrialByNameNoScope(program, observationUnitByNameNoScope, experimentImportRows); + Map> studyByNameNoScope = initializeStudyByNameNoScope(program, trialByNameNoScope, observationUnitByNameNoScope, experimentImportRows); + // interesting we're using our data model instead of brapi for locations + Map> locationByName = initializeUniqueLocationNames(program, studyByNameNoScope, experimentImportRows); + Map> obsVarDatasetByName = initializeObsVarDatasetByName(program, trialByNameNoScope, experimentImportRows); + Map> existingGermplasmByGID = initializeExistingGermplasmByGID(program, observationUnitByNameNoScope, experimentImportRows); + + PendingData existing = PendingData.builder() + .observationUnitByNameNoScope(observationUnitByNameNoScope) + .trialByNameNoScope(trialByNameNoScope) + .studyByNameNoScope(studyByNameNoScope) + .locationByName(locationByName) + .obsVarDatasetByName(obsVarDatasetByName) + .existingGermplasmByGID(existingGermplasmByGID) + .build(); + + return existing; + } + + /** + * Initializes the observation units for the given program and experimentImportRows. + * + * @param program The program object + * @param experimentImportRows A list of ExperimentObservation objects + * @return A map of Observation Unit IDs to PendingImportObject objects + * + * @throws InternalServerException + * @throws IllegalStateException + */ + private Map> initializeObservationUnits(Program program, List experimentImportRows) { + Map> observationUnitByName = new HashMap<>(); + + Map rowByObsUnitId = new HashMap<>(); + experimentImportRows.forEach(row -> { + if (StringUtils.isNotBlank(row.getObsUnitID())) { + if(rowByObsUnitId.containsKey(row.getObsUnitID())) { + throw new IllegalStateException("ObsUnitId is repeated: " + row.getObsUnitID()); + } + rowByObsUnitId.put(row.getObsUnitID(), row); + } + }); + + try { + List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); + + // TODO: grab from externalReferences + /* + observationUnitByObsUnitId = existingObsUnits.stream() + .collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, + (BrAPIObservationUnit unit) -> new PendingImportObject<>(unit, false))); + */ + + String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + if (existingObsUnits.size() == rowByObsUnitId.size()) { + existingObsUnits.forEach(brAPIObservationUnit -> { + processAndCacheObservationUnit(brAPIObservationUnit, refSource, program, observationUnitByName, rowByObsUnitId); + + BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) + .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); + + ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); + row.setExpTitle(Utilities.removeProgramKey(brAPIObservationUnit.getTrialName(), program.getKey())); + row.setEnv(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getStudyName(), program.getKey())); + row.setEnvLocation(Utilities.removeProgramKey(brAPIObservationUnit.getLocationName(), program.getKey())); + }); + } else { + List missingIds = new ArrayList<>(rowByObsUnitId.keySet()); + missingIds.removeAll(existingObsUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList())); + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + + return observationUnitByName; + } catch (ApiException e) { + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + /** + * Adds a new map entry to observationUnitByName based on the brAPIObservationUnit passed in and sets the + * expUnitId in the rowsByObsUnitId map. + * + * @param brAPIObservationUnit the BrAPI observation unit object + * @param refSource the reference source + * @param program the program object + * @param observationUnitByName the map of observation units by name (will be modified in place) + * @param rowByObsUnitId the map of rows by observation unit ID (will be modified in place) + * + * @throws InternalServerException + */ + private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservationUnit, String refSource, Program program, + Map> observationUnitByName, + Map rowByObsUnitId) { + BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) + .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); + + ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); + row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); + observationUnitByName.put(createObservationUnitKey(row), + new PendingImportObject<>(ImportObjectState.EXISTING, + brAPIObservationUnit, + UUID.fromString(idRef.getReferenceId()))); + } + + private String createObservationUnitKey(ExperimentObservation importRow) { + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); + } + + private String createObservationUnitKey(String studyName, String obsUnitName) { + return studyName + obsUnitName; + } + + /** + * Initializes studies by name without scope. + * + * @param program The program object. + * @param trialByNameNoScope A map of trial names with their corresponding pending import objects. + * @param experimentImportRows A list of experiment observation objects. + * @return A map of study names with their corresponding pending import objects. + * @throws InternalServerException If there is an error while processing the method. + */ + private Map> initializeStudyByNameNoScope(Program program, + Map> trialByNameNoScope, + Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> studyByName = new HashMap<>(); + if (trialByNameNoScope.size() != 1) { + return studyByName; + } + + try { + initializeStudiesForExistingObservationUnits(program, studyByName, observationUnitByNameNoScope); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing studies", e); + throw new InternalServerException(e.toString(), e); + } + + List existingStudies; + Optional> trial = getTrialPIO(experimentImportRows, trialByNameNoScope); + + try { + if (trial.isEmpty()) { + // TODO: throw ValidatorException and return 422 + } + UUID experimentId = trial.get().getId(); + existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); + for (BrAPIStudy existingStudy : existingStudies) { + studyService.processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); + } + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing studies: ", e); + throw new InternalServerException(e.toString(), e); + } + + return studyByName; + } + + /** + * Retrieves the PendingImportObject of a BrAPITrial based on the given list of ExperimentObservation and trialByNameNoScope map. + * + * @param experimentImportRows The list of ExperimentObservation objects. + * @param trialByNameNoScope The map of trial names to PendingImportObject of BrAPITrial. + * @return The Optional containing the PendingImportObject of BrAPITrial, or an empty Optional if no matching trial is found. + */ + private Optional> getTrialPIO(List experimentImportRows, + Map> trialByNameNoScope) { + Optional expTitle = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID()) && StringUtils.isNotBlank(row.getExpTitle())) + .map(ExperimentObservation::getExpTitle) + .findFirst(); + + if (expTitle.isEmpty() && trialByNameNoScope.keySet().stream().findFirst().isEmpty()) { + return Optional.empty(); + } + if(expTitle.isEmpty()) { + expTitle = trialByNameNoScope.keySet().stream().findFirst(); + } + + return Optional.ofNullable(trialByNameNoScope.get(expTitle.get())); + } + + + private void initializeStudiesForExistingObservationUnits( + Program program, + Map> studyByName, + Map> observationUnitByNameNoScope + ) throws Exception { + Set studyDbIds = observationUnitByNameNoScope.values() + .stream() + .map(pio -> pio.getBrAPIObject() + .getStudyDbId()) + .collect(Collectors.toSet()); + + List studies = fetchStudiesByDbId(studyDbIds, program); + for (BrAPIStudy study : studies) { + studyService.processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); + } + } + + /** + * Initializes unique location names for a program. + * + * @param program The program object. + * @param studyByNameNoScope A map of study names and corresponding BrAPI study objects. + * @param experimentImportRows A list of experiment observation objects for import. + * @return A map of location names and their corresponding pending import objects. + * @throws InternalServerException If there is an error fetching locations. + */ + private Map> initializeUniqueLocationNames(Program program, + Map> studyByNameNoScope, + List experimentImportRows) { + Map> locationByName = new HashMap<>(); + + List existingLocations = new ArrayList<>(); + if(studyByNameNoScope.size() > 0) { + Set locationDbIds = studyByNameNoScope.values() + .stream() + .map(study -> study.getBrAPIObject() + .getLocationDbId()) + .collect(Collectors.toSet()); + try { + existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueLocationNames = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getEnvLocation) + .distinct() + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + try { + existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); + return locationByName; + } + + /** + * Initializes observation variable dataset by name. + * + * @param program The program associated with the dataset. + * @param trialByNameNoScope The map of trials identified by name without scope. + * @param experimentImportRows The list of experiment observation rows. + * @return The map of observation variable dataset indexed by name. + * + * @throws InternalServerException + */ + private Map> initializeObsVarDatasetByName(Program program, + Map> trialByNameNoScope, + List experimentImportRows) { + Map> obsVarDatasetByName = new HashMap<>(); + + Optional> trialPIO = getTrialPIO(experimentImportRows, trialByNameNoScope); + + if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String datasetId = trialPIO.get().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + try { + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(datasetId)); + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("existing dataset summary not returned from brapi server"); + } + BrAPIListDetails dataSetDetails = brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult(); + processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + return obsVarDatasetByName; + } + + /** + * Process and cache an object of type BrAPIListDetails. + * + * @param existingList The existing list to be processed and cached + * @param obsVarDatasetByName A map of ObsVarDatasets indexed by name (will be modified in place) + * + * @throws IllegalStateException + */ + private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map> obsVarDatasetByName) { + BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for list (dbid): " + existingList.getListDbId())); + obsVarDatasetByName.put(existingList.getListName(), + new PendingImportObject<>(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceId()))); + } + + /** + * Initializes existing germplasm objects by germplasm ID (GID). + * + * @param program The program object. + * @param observationUnitByNameNoScope A map of observation unit objects by name. + * @param experimentImportRows A list of experiment observation objects. + * @return A map of existing germplasm objects by germplasm ID. + * + * @throws InternalServerException + */ + private Map> initializeExistingGermplasmByGID(Program program, + Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> existingGermplasmByGID = new HashMap<>(); + + List existingGermplasms = new ArrayList<>(); + if(observationUnitByNameNoScope.size() > 0) { + Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + try { + existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueGermplasmGIDs = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getGid) + .distinct() + .collect(Collectors.toList()); + + try { + existingGermplasms.addAll(getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + existingGermplasms.forEach(existingGermplasm -> { + BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); + }); + return existingGermplasmByGID; + } + + /** + * Retrieves a list of germplasm with the specified accession numbers. + * + * @param germplasmAccessionNumbers The list of accession numbers to search for. + * @param programId The ID of the program. + * @return An ArrayList of BrAPIGermplasm objects that match the accession numbers. + * @throws ApiException if there is an error retrieving the germplasm. + */ + private ArrayList getGermplasmByAccessionNumber( + List germplasmAccessionNumbers, + UUID programId) throws ApiException { + List germplasmList = brAPIGermplasmDAO.getGermplasm(programId); + ArrayList resultGermplasm = new ArrayList<>(); + // Search for accession number matches + for (BrAPIGermplasm germplasm : germplasmList) { + for (String accessionNumber : germplasmAccessionNumbers) { + if (germplasm.getAccessionNumber() + .equals(accessionNumber)) { + resultGermplasm.add(germplasm); + break; + } + } + } + return resultGermplasm; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java new file mode 100644 index 000000000..cfab4b5a1 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java @@ -0,0 +1,16 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.steps; + +import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; + +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; + +public class ProcessStep implements ProcessingStep { + + @Override + public ProcessedData process(PendingData input) { + + // TODO: implement + return new ProcessedData(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java new file mode 100644 index 000000000..e2446ecef --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -0,0 +1,70 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.steps.GetExistingProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.steps.ProcessStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.Pipeline; + +import javax.inject.Inject; +import javax.inject.Named; +import javax.inject.Provider; + +/** + * This class represents a workflow for creating a new experiment. The bean name must match the appropriate bean column + * value in the import_mapping_workflow db table + */ +@Prototype +@Named("CreateNewExperimentWorkflow") +public class CreateNewExperimentWorkflow implements Workflow { + + private final Provider getExistingStepProvider; + private final Provider processStepProvider; + + @Inject + public CreateNewExperimentWorkflow(Provider getExistingStepProvider, + Provider processStepProvider) { + this.getExistingStepProvider = getExistingStepProvider; + this.processStepProvider = processStepProvider; + } + + @Override + public ProcessedData process(ImportContext context) { + // TODO + Pipeline pipeline = new Pipeline<>(getExistingStepProvider.get()) + .addProcessingStep(processStepProvider.get()); + ProcessedData processed = pipeline.execute(context); + + // TODO: return actual data + return processed; + } + + /** + * Retrieves the name of the workflow. This is used for logging display purposes. + * + * @return the name of the workflow + */ + @Override + public String getName() { + return "CreateNewExperimentWorkflow"; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java new file mode 100644 index 000000000..c3c78cdd6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java @@ -0,0 +1,60 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.middleware; + +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; + +public abstract class Middleware { + + Middleware next; + Middleware prior; + + /** + * Builds chains of middleware objects. + */ + public static Middleware link(Middleware first, Middleware... chain) { + Middleware head = first; + for (Middleware nextInChain: chain) { + nextInChain.prior = head.getLastLink(); + head.getLastLink().next = nextInChain; + head = nextInChain; + } + return first; + } + + /** + * Subclasses will implement this local transaction. + */ + public abstract boolean process(T context); + /** + * Subclasses will implement this method to handle errors and possibly undo the local transaction. + */ + public abstract boolean compensate(T context, MiddlewareError error); + /** + * Processes the next local transaction or ends traversing if we're at the + * last local transaction of the transaction. + */ + protected boolean processNext(T context) { + if (next == null) { + return true; + } + return next.process(context); + } + + /** + * Runs the compensating local transaction for the prior local transaction or ends traversing if + * we're at the first local transaction of the transaction. + */ + protected boolean compensatePrior(T context, MiddlewareError error) { + if (prior == null) { + return true; + } + return prior.compensate(context, error); + } + + private Middleware getLastLink() { + return this.next == null ? this : this.next.getLastLink(); + } + + private Middleware getFirstLink() { + return this.prior == null ? this : this.prior.getFirstLink(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java new file mode 100644 index 000000000..0fc8b807c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java @@ -0,0 +1,31 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import com.fasterxml.jackson.annotation.JsonValue; +import io.micronaut.context.annotation.Property; + +public class ExpImportProcessConstants { + + public static final CharSequence COMMA_DELIMITER = ","; + public static final String TIMESTAMP_PREFIX = "TS:"; + public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; + public static final String MIDNIGHT = "T00:00:00-00:00"; + @Property(name = "brapi.server.reference-source") + public static String BRAPI_REFERENCE_SOURCE; + public enum ErrMessage { + MULTIPLE_EXP_TITLES("File contains more than one Experiment Title"), + MISSING_OBS_UNIT_ID_ERROR("Experimental entities are missing ObsUnitIDs"), + PREEXISTING_EXPERIMENT_TITLE("Experiment Title already exists"); + + private String value; + + ErrMessage(String value) { + this.value = value; + } + + @JsonValue + public String getValue() { + return value; + } + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java new file mode 100644 index 000000000..1e81ff833 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java @@ -0,0 +1,13 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import lombok.*; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; + +@Data +@Builder +public class ExpUnitMiddlewareContext { + private ImportContext importContext; + private ExpUnitContext expUnitContext; + private PendingData pendingData; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ImportContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ImportContext.java new file mode 100644 index 000000000..0738d6a1e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ImportContext.java @@ -0,0 +1,28 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import tech.tablesaw.api.Table; + +import java.util.List; +import java.util.Map; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ImportContext { + private ImportUpload upload; + private List importRows; + private Map mappedBrAPIImport; + private Table data; + private Program program; + private User user; + private boolean commit; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java new file mode 100644 index 000000000..49960ae3a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java @@ -0,0 +1,21 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import lombok.Getter; +import lombok.Setter; + +public class MiddlewareError { + @Getter + @Setter + String localTransactionName; + Runnable handler; + + public MiddlewareError(Runnable handler) { + this.handler = handler; + } + + public void tag(String name) { + if (this.getLocalTransactionName() == null) { + this.setLocalTransactionName(name); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java new file mode 100644 index 000000000..49fe30b12 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java @@ -0,0 +1,14 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; + +import java.util.Map; + +@Data +@Builder +@ToString +@NoArgsConstructor +public class ProcessedData { + private Map statistics; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java new file mode 100644 index 000000000..5776ab59b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java @@ -0,0 +1,50 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; + +import javax.inject.Named; + +/** + * This class represents a workflow for adding new environments to an existing experiment. The bean name must match + * the appropriate bean column value in the import_mapping_workflow db table + */ + +@Prototype +@Named("CreateNewEnvironmentWorkflow") +public class CreateNewEnvironmentWorkflow implements Workflow { + @Override + public ProcessedData process(ImportContext context) { + // TODO + return null; + } + + /** + * Retrieves the name of the workflow. This is used for logging display purposes. + * + * @return the name of the workflow + */ + @Override + public String getName() { + return "CreateNewEnvironmentWorkflow"; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/Pipeline.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/Pipeline.java new file mode 100644 index 000000000..09b657a89 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/Pipeline.java @@ -0,0 +1,18 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.pipeline; + +public class Pipeline { + + private final ProcessingStep currentStep; + + public Pipeline(ProcessingStep currentStep) { + this.currentStep = currentStep; + } + + public Pipeline addProcessingStep(ProcessingStep newStep) { + return new Pipeline<>(input -> newStep.process(currentStep.process(input))); + } + + public O execute(I input) { + return currentStep.process(input); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/ProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/ProcessingStep.java new file mode 100644 index 000000000..2407e646e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/ProcessingStep.java @@ -0,0 +1,5 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.pipeline; + +public interface ProcessingStep { + O process(I input); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java new file mode 100644 index 000000000..399f23468 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -0,0 +1,312 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.BrAPIListTypes; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.request.BrAPIListNewRequest; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +public class DatasetService { + private final BrAPIListDAO brAPIListDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public DatasetService(BrAPIListDAO brapiListDAO) { + this.brAPIListDAO = brapiListDAO; + } + /** + * Module: Dataset Utility + * + * This module provides utility functions for interacting with datasets using the BrAPI standards. + * It includes methods for fetching dataset details, creating new datasets, updating existing datasets, etc. + * Usage: This module can be used in various applications where handling BrAPI-compliant datasets is required. + */ + + /** + * Fetches dataset details by dataset ID and program + * + * This function fetches details of a dataset by its ID and associated program from a data source using the BrAPI standards. + * + * @param id The unique identifier of the dataset to fetch + * @param program The program object associated with the dataset + * @return BrAPIListDetails object containing the details of the dataset + * @throws ApiException if there is an issue with fetching the dataset details from the data source + */ + public Optional fetchDatasetById(String id, Program program) throws ApiException { + Optional dataSetDetails = Optional.empty(); + + // Retrieve existing dataset summaries based on program ID and external reference + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(id)); + + // Check if the existing dataset summaries are returned, throw exception if not + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("Existing dataset summary not returned from BrAPI server"); + } + + // Retrieve dataset details using the list DB ID from the existing dataset summary + dataSetDetails = Optional.ofNullable(brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult()); + + return dataSetDetails; + } + + /** + * Constructs a PendingImportObject for a BrAPIListDetails dataset. + * This method retrieves the external reference for the dataset from the existing list + * based on a specific reference source. It then creates a PendingImportObject for the dataset + * with the existing list and reference ID. + * + * @param dataset The BrAPIListDetails dataset for which to construct the PendingImportObject + * @param program + * @return A PendingImportObject containing the dataset with the existing list and reference ID + * @throws IllegalStateException if external references weren't found for the list + */ + public PendingImportObject constructPIOFromDataset(BrAPIListDetails dataset, Program program) { + // Get the external reference for the dataset from the existing list + BrAPIExternalReference xref = Utilities.getExternalReference(dataset.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) + .orElseThrow(() -> new IllegalStateException("External references weren't found for list (dbid): " + dataset.getListDbId())); + + // Create a PendingImportObject for the dataset with the existing list and reference ID + return new PendingImportObject(ImportObjectState.EXISTING, dataset, UUID.fromString(xref.getReferenceId())); + } + + // TODO: used by expunit worflow + public Map> initializeObsVarDatasetForExistingObservationUnits( + Map> trialByName, + Program program) { + Map> obsVarDatasetByName = new HashMap<>(); + + if (trialByName.size() > 0 && + trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String datasetId = trialByName.values().iterator().next().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + + try { + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(datasetId)); + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("existing dataset summary not returned from brapi server"); + } + BrAPIListDetails dataSetDetails = brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult(); + processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + return obsVarDatasetByName; + } + + // TODO: used by create workflow + public Map> initializeObsVarDatasetByName(Program program, List experimentImportRows) { + Map> obsVarDatasetByName = new HashMap<>(); + + Optional> trialPIO = getTrialPIO(experimentImportRows); + + if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String datasetId = trialPIO.get().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + try { + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(datasetId)); + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("existing dataset summary not returned from brapi server"); + } + BrAPIListDetails dataSetDetails = brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult(); + processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + return obsVarDatasetByName; + } + + // TODO: used by expunit workflow + public Map> mapPendingObsDatasetByOUId( + String unitId, + Map> trialByOUId, + Map> obsVarDatasetByName, + Map> obsVarDatasetByOUId) { + if (!trialByOUId.isEmpty() && !obsVarDatasetByName.isEmpty() && + trialByOUId.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + obsVarDatasetByOUId.put(unitId, obsVarDatasetByName.values().iterator().next()); + } + + return obsVarDatasetByOUId; + } + + // TODO: used by both workflows + public void addObsVarsToDatasetDetails(PendingImportObject pio, List referencedTraits, Program program) { + BrAPIListDetails details = pio.getBrAPIObject(); + referencedTraits.forEach(trait -> { + String id = Utilities.appendProgramKey(trait.getObservationVariableName(), program.getKey()); + + // TODO - Don't append the key if connected to a brapi service operating with legacy data(no appended program key) + + if (!details.getData().contains(id) && ImportObjectState.EXISTING != pio.getState()) { + details.getData().add(id); + } + if (!details.getData().contains(id) && ImportObjectState.EXISTING == pio.getState()) { + details.getData().add(id); + pio.setState(ImportObjectState.MUTATED); + } + }); + } + + // TODO: used by expunit workflow + public void fetchOrCreateDatasetPIO(ImportContext importContext, + PendingData pendingData, + ExpUnitContext expUnitContext, + List referencedTraits) throws UnprocessableEntityException { + PendingImportObject pio; + PendingImportObject trialPIO = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); + String name = String.format("Observation Dataset [%s-%s]", + program.getKey(), + trialPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString()); + if (obsVarDatasetByName.containsKey(name)) { + pio = obsVarDatasetByName.get(name); + } else { + UUID id = UUID.randomUUID(); + BrAPIListDetails newDataset = importRow.constructDatasetDetails( + name, + id, + BRAPI_REFERENCE_SOURCE, + program, + trialPIO.getId().toString()); + pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); + trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); + if (ImportObjectState.EXISTING == trialPIO.getState()) { + trialPIO.setState(ImportObjectState.MUTATED); + } + obsVarDatasetByName.put(name, pio); + } + addObsVarsToDatasetDetails(pio, referencedTraits, program); + } + + // TODO: used by create workflow + public void fetchOrCreateDatasetPIO(ImportContext importContext, + PendingData pendingData, + List referencedTraits) throws UnprocessableEntityException { + PendingImportObject pio; + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); + + String name = String.format("Observation Dataset [%s-%s]", + program.getKey(), + trialPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString()); + if (obsVarDatasetByName.containsKey(name)) { + pio = obsVarDatasetByName.get(name); + } else { + UUID id = UUID.randomUUID(); + BrAPIListDetails newDataset = importRow.constructDatasetDetails( + name, + id, + BRAPI_REFERENCE_SOURCE, + program, + trialPIO.getId().toString()); + pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); + trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); + if (ImportObjectState.EXISTING == trialPIO.getState()) { + trialPIO.setState(ImportObjectState.MUTATED); + } + obsVarDatasetByName.put(name, pio); + } + addObsVarsToDatasetDetails(pio, referencedTraits, program); + } + + // TODO: used by both workflows + public List commitNewPendingDatasetsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List newDatasetRequests = ProcessorData.getNewObjects(obsVarDatasetByName).stream().map(details -> { + BrAPIListNewRequest request = new BrAPIListNewRequest(); + request.setListName(details.getListName()); + request.setListType(details.getListType()); + request.setExternalReferences(details.getExternalReferences()); + request.setAdditionalInfo(details.getAdditionalInfo()); + request.data(details.getData()); + return request; + }).collect(Collectors.toList()); + List createdDatasets = new ArrayList<>(brAPIListDAO.createBrAPILists(newDatasetRequests, program.getId(), upload)); + for (BrAPIListSummary summary : createdDatasets) { + obsVarDatasetByName.get(summary.getListName()).getBrAPIObject().setListDbId(summary.getListDbId()); + } + return createdDatasets; + } + + // TODO: used by both workflows + public List commitUpdatedPendingDatasetsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedDatasets = new ArrayList<>(); + Map datasetNewDataById = ProcessorData + .getMutationsByObjectId(obsVarDatasetByName, BrAPIListSummary::getListDbId); + for (Map.Entry entry : datasetNewDataById.entrySet()) { + String id = entry.getKey(); + BrAPIListDetails dataset = entry.getValue(); + try { + List existingObsVarIds = brAPIListDAO.getListById(id, program.getId()).getResult().getData(); + List newObsVarIds = dataset + .getData() + .stream() + .filter(obsVarId -> !existingObsVarIds.contains(obsVarId)).collect(Collectors.toList()); + List obsVarIds = new ArrayList<>(existingObsVarIds); + obsVarIds.addAll(newObsVarIds); + dataset.setData(obsVarIds); + updatedDatasets.add(brAPIListDAO.updateBrAPIList(id, dataset, program.getId())); + } catch (ApiException e) { + log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating dataset observation variables: ", e); + throw new InternalServerException(e.getMessage(), e); + } + } + return updatedDatasets; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java new file mode 100644 index 000000000..b4e7127cd --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java @@ -0,0 +1,152 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import java.util.*; +import java.util.stream.Collectors; + +public class GermplasmService { + private final BrAPIGermplasmDAO germplasmDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + public GermplasmService(BrAPIGermplasmDAO germplasmDAO) { + this.germplasmDAO = germplasmDAO; + } + + /** + * Retrieves a list of BrAPI Germplasm objects based on the provided set of database IDs and a Program object. + * + * @param dbIds A Set of database IDs (strings) used to filter germplasm data retrieval. + * @param program The Program object representing the program associated with the germplasm data. + * @return A List of BrAPIGermplasm objects that match the provided database IDs and program. + * @throws ApiException If an error occurs during the retrieval process. + */ + public List fetchGermplasmByDbId(Set dbIds, Program program) throws ApiException { + List brapiGermplasm = null; + brapiGermplasm = germplasmDAO.getGermplasmsByDBID(dbIds, program.getId()); + return brapiGermplasm; + } + + /** + * This method constructs a PendingImportObject for a given BrAPI Germplasm. + * It retrieves the External Reference associated with the Germplasm and constructs a PendingImportObject with ImportObjectState set to EXISTING. + * + * @param brapiGermplasm The BrAPI Germplasm object for which the PendingImportObject needs to be constructed + * @return PendingImportObject A PendingImportObject containing the BrAPI Germplasm object and its External Reference + * @throws IllegalStateException if the External Reference for the Germplasm is not found + */ + public PendingImportObject constructPIOFromBrapiGermplasm(BrAPIGermplasm brapiGermplasm) { + // Initialize the PendingImportObject to null + PendingImportObject pio = null; + + // Retrieve the External Reference associated with the Germplasm from the Utilities class + BrAPIExternalReference xref = Utilities.getExternalReference(brapiGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + // Throw an exception if External Reference is not found + .orElseThrow(() -> new IllegalStateException("External references weren't found for germplasm (dbid): " + brapiGermplasm.getGermplasmDbId())); + + // Construct the PendingImportObject with ImportObjectState set to EXISTING and External Reference UUID + pio = new PendingImportObject<>(ImportObjectState.EXISTING, brapiGermplasm, UUID.fromString(xref.getReferenceId())); + + return pio; + } + + /** + * Retrieves the Germplasm ID from a PendingImportObject containing BrAPI Germplasm data. + * This method extracts the Germplasm ID (GID) from the Accession Number of the BrAPI Germplasm object within the PendingImportObject. + * + * @param pio a PendingImportObject that wraps BrAPI Germplasm data + * @return a String representing the Germplasm ID extracted from the Accession Number + */ + public String getGIDFromGermplasmPIO(PendingImportObject pio) { + String gid = null; + gid = pio.getBrAPIObject().getAccessionNumber(); + return gid; + } + + // TODO: used by expunit workflow + public Map> initializeGermplasmByGIDForExistingObservationUnits( + Map> unitByName, + Program program) { + Map> existingGermplasmByGID = new HashMap<>(); + + List existingGermplasms = new ArrayList<>(); + if(unitByName.size() > 0) { + Set germplasmDbIds = unitByName.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + try { + existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + existingGermplasms.forEach(existingGermplasm -> { + BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); + }); + return existingGermplasmByGID; + } + + // TODO: used by create worflow + public Map> initializeExistingGermplasmByGID(Program program, List experimentImportRows) { + Map> existingGermplasmByGID = new HashMap<>(); + + List existingGermplasms = new ArrayList<>(); + if(observationUnitByNameNoScope.size() > 0) { + Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + try { + existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueGermplasmGIDs = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getGid) + .distinct() + .collect(Collectors.toList()); + + try { + existingGermplasms.addAll(this.getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + existingGermplasms.forEach(existingGermplasm -> { + BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); + }); + return existingGermplasmByGID; + } + + // TODO: used by expunit workflow + public Map> mapGermplasmByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> germplasmByName, + Map> germplasmByOUId) { + String gid = unit.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.GID).getAsString(); + germplasmByOUId.put(unitId, germplasmByName.get(gid)); + + return germplasmByOUId; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java new file mode 100644 index 000000000..2fe6815ca --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java @@ -0,0 +1,212 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.ProgramLocationService; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; + +@Singleton +@Slf4j +public class LocationService { + private final ProgramLocationService programLocationService; + public LocationService(ProgramLocationService programLocationService) { + this.programLocationService = programLocationService; + } + + /** + * Fetches a list of ProgramLocation objects based on the provided locationDbIds and program ID. + * + * @param locationDbIds A set of locationDbIds used to query locations. + * @param program The Program object for which locations are being fetched. + * @return A list of ProgramLocation objects matching the given locationDbIds for the program. + * @throws ApiException if there is an issue with fetching the locations or if any location(s) are not found. + */ + public List fetchLocationsByDbId(Set locationDbIds, Program program) throws ApiException { + List programLocations = null; // Initializing the ProgramLocations list + + // Retrieving locations based on the locationDbId and the program's ID + programLocations = programLocationService.getLocationsByDbId(locationDbIds, program.getId()); + + // If no locations are found, throw an IllegalStateException with an error message + if (locationDbIds.size() != programLocations.size()) { + Set missingIds = new HashSet<>(locationDbIds); + missingIds.removeAll(programLocations.stream().map(ProgramLocation::getLocationDbId).collect(Collectors.toSet())); + throw new IllegalStateException("Location not found for location dbid(s): " + String.join(COMMA_DELIMITER, missingIds)); + } + + // Return the fetched ProgramLocations + return programLocations; + } + + /** + * Constructs a PendingImportObject of type ProgramLocation from a given BrAPI ProgramLocation. + * This method creates a new PendingImportObject with the state set to EXISTING and the BrAPI ProgramLocation as the source object. + * + * @param brapiLocation The BrAPI ProgramLocation from which the PendingImportObject should be constructed + * @return PendingImportObject The PendingImportObject created from the BrAPI ProgramLocation + */ + public PendingImportObject constructPIOFromBrapiLocation(ProgramLocation brapiLocation) { + return new PendingImportObject<>(ImportObjectState.EXISTING, brapiLocation, brapiLocation.getId()); + } + + // used by expunit workflow + public Map> initializeLocationByName( + Program program, + Map> studyByName) { + Map> locationByName = new HashMap<>(); + + List existingLocations = new ArrayList<>(); + if(studyByName.size() > 0) { + Set locationDbIds = studyByName.values() + .stream() + .map(study -> study.getBrAPIObject() + .getLocationDbId()) + .collect(Collectors.toSet()); + try { + existingLocations.addAll(programLocationService.getLocationsByDbId(locationDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + existingLocations.forEach(existingLocation -> locationByName.put( + existingLocation.getName(), + new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()) + ) + ); + return locationByName; + } + + // TODO: used by create workflow + public Map> initializeUniqueLocationNames(Program program, List experimentImportRows) { + Map> locationByName = new HashMap<>(); + + List existingLocations = new ArrayList<>(); + if(studyByNameNoScope.size() > 0) { + Set locationDbIds = studyByNameNoScope.values() + .stream() + .map(study -> study.getBrAPIObject() + .getLocationDbId()) + .collect(Collectors.toSet()); + try { + existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueLocationNames = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getEnvLocation) + .distinct() + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + try { + existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); + return locationByName; + } + + // TODO: used by expunit workflow + public Map> mapPendingLocationByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> studyByOUId, + Map> locationByName, + Map> locationByOUId + ) { + if (unit.getLocationName() != null) { + locationByOUId.put(unitId, locationByName.get(unit.getLocationName())); + } else if (studyByOUId.get(unitId) != null && studyByOUId.get(unitId).getBrAPIObject().getLocationName() != null) { + locationByOUId.put( + unitId, + locationByName.get(studyByOUId.get(unitId).getBrAPIObject().getLocationName()) + ); + } else { + throw new IllegalStateException("Observation unit missing location: " + unitId); + } + + return locationByOUId; + } + + // TODO: used by expunit workflow + private void fetchOrCreateLocationPIO(ImportContext importContext, ExpUnitContext expUnitContext) { + PendingImportObject pio; + String envLocationName = pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName(); + if (!locationByName.containsKey((importRow.getEnvLocation()))) { + ProgramLocation newLocation = new ProgramLocation(); + newLocation.setName(envLocationName); + pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); + this.locationByName.put(envLocationName, pio); + } + } + + // TODO: used by create workflow + private void fetchOrCreateLocationPIO(ImportContext importContext) { + PendingImportObject pio; + String envLocationName = importRow.getEnvLocation(); + if (!locationByName.containsKey((importRow.getEnvLocation()))) { + ProgramLocation newLocation = new ProgramLocation(); + newLocation.setName(envLocationName); + pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); + this.locationByName.put(envLocationName, pio); + } + } + + // TODO: used by both workflows + public List commitNewPendingLocationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + AuthenticatedUser actingUser = new AuthenticatedUser(upload.getUpdatedByUser().getName(), new ArrayList<>(), upload.getUpdatedByUser().getId(), new ArrayList<>()); + + List newLocations = ProcessorData.getNewObjects(this.locationByName) + .stream() + .map(location -> ProgramLocationRequest.builder() + .name(location.getName()) + .build()) + .collect(Collectors.toList()); + + List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); + // set the DbId to the for each newly created location + for (ProgramLocation createdLocation : createdLocations) { + String createdLocationName = createdLocation.getName(); + this.locationByName.get(createdLocationName) + .getBrAPIObject() + .setLocationDbId(createdLocation.getLocationDbId()); + } + return createdLocations; + } + + // TODO: used by both workflows + public List commitUpdatedPendingLocationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedLocations = new ArrayList<>(); + + return updatedLocations; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java new file mode 100644 index 000000000..641aa153d --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -0,0 +1,323 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.OverwrittenData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Scale; +import org.breedinginsight.model.Trait; +import org.breedinginsight.utilities.Utilities; +import tech.tablesaw.columns.Column; + +import java.math.BigDecimal; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.*; +import java.util.stream.Collectors; + +public class ObservationService { + public boolean validCategory(List categories, String value) { + Set categoryValues = categories.stream() + .map(category -> category.getValue().toLowerCase()) + .collect(Collectors.toSet()); + return categoryValues.contains(value.toLowerCase()); + } + public boolean validNumericRange(BigDecimal value, Scale validValues) { + // account for empty min or max in valid determination + return (validValues.getValidValueMin() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMin())) >= 0) && + (validValues.getValidValueMax() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMax())) <= 0); + } + public Optional validNumericValue(String value) { + BigDecimal number; + try { + number = new BigDecimal(value); + } catch (NumberFormatException e) { + return Optional.empty(); + } + return Optional.of(number); + } + + public boolean isBlankObservation(String value) { + return StringUtils.isBlank(value); + } + public boolean isNAObservation(String value){ + return value.equalsIgnoreCase("NA"); + } + public boolean validDateTimeValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } + + public boolean validDateValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } + public String getObservationHash(String observationUnitName, String variableName, String studyName) { + String concat = DigestUtils.sha256Hex(observationUnitName) + + DigestUtils.sha256Hex(variableName) + + DigestUtils.sha256Hex(StringUtils.defaultString(studyName)); + return DigestUtils.sha256Hex(concat); + } + public Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { + Set ouDbIds = new HashSet<>(); + Set variableDbIds = new HashSet<>(); + Map variableNameByDbId = new HashMap<>(); + Map ouNameByDbId = new HashMap<>(); + Map studyNameByDbId = studyByNameNoScope.values() + .stream() + .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) + .map(PendingImportObject::getBrAPIObject) + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); + + studyNameByDbId.keySet().forEach(studyDbId -> { + try { + brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { + if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { + ouDbIds.add(ou.getObservationUnitDbId()); + } + ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + }); + } catch (ApiException e) { + throw new RuntimeException(e); + } + }); + + for (Trait referencedTrait : referencedTraits) { + variableDbIds.add(referencedTrait.getObservationVariableDbId()); + variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); + } + + List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); + + return existingObservations.stream() + .map(obs -> { + String studyName = studyNameByDbId.get(obs.getStudyDbId()); + String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); + String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); + + String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); + + return Map.entry(key, obs); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + // TODO: used with expUnit workflow + public void validateObservations(PendingData pendingData, + int rowNum, + ImportContext importContext, + ExpUnitContext expUnitContext, + List> phenotypeCols, + CaseInsensitiveMap colVarMap) { + for (Column phenoCol : phenotypeCols) { + String importHash; + String importObsValue = phenoCol.getString(rowNum); + + importHash = getImportObservationHash( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(phenoCol), + pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() + ); + + validateObservation(importHash); + } + } + + // TODO: used with create workflow + public void validateObservations(PendingData pendingData, + int rowNum, + ImportContext importContext, + List> phenotypeCols, + CaseInsensitiveMap colVarMap) { + + + for (Column phenoCol : phenotypeCols) { + String importHash; + String importObsValue = phenoCol.getString(rowNum); + + importHash = getImportObservationHash(importRow, phenoCol.name()); + + validateObservation(importHash); + } + + } + + // TODO: used by both workflows + private void validateObservation(String importHash) { + + + String importObsValue = phenoCol.getString(rowNum); + + + // error if import observation data already exists and user has not selected to overwrite + if (commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && + this.existingObsByObsHash.containsKey(importHash) && + StringUtils.isNotBlank(phenoCol.getString(rowNum)) && + !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { + addRowError( + phenoCol.name(), + String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), + validationErrors, rowNum + ); + + // preview case where observation has already been committed and the import row ObsVar data differs from what + // had been saved prior to import + } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { + + // add a change log entry when updating the value of an observation + if (commit) { + BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; + String prior = ""; + if (isValueMatched(importHash, importObsValue)) { + prior.concat(existingObsByObsHash.get(importHash).getValue()); + } + if (timeStampColByPheno.containsKey(phenoCol.name()) && isTimestampMatched(importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { + prior = prior.isEmpty() ? prior : prior.concat(" "); + prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); + } + ChangeLogEntry change = new ChangeLogEntry(prior, + reason, + user.getId(), + timestamp + ); + + // create the changelog field in additional info if it does not already exist + OverwrittenData.createChangeLog(pendingObservation); + + // add a new entry to the changelog + pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); + } + + // preview case where observation has already been committed and import ObsVar data is the + // same as has been committed prior to import + } else if (isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { + BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); + existingObs.setObservationVariableName(phenoCol.name()); + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + observationByHash.get(importHash).setBrAPIObject(existingObs); + + // preview case where observation has already been committed and import ObsVar data is empty prior to import + } else if (!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + } else { + validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); + + //Timestamp validation + if (timeStampColByPheno.containsKey(phenoCol.name())) { + Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); + validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); + } + } + + } + + // TODO: used by both workflows + private void updateObservationDependencyValues(Program program) { + String programKey = program.getKey(); + + // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds + this.observationUnitByNameNoScope.values().stream() + .map(PendingImportObject::getBrAPIObject) + .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); + + // Update ObservationVariable DbIds + List traits = getTraitList(program); + CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); + for ( Trait trait: traits) { + traitMap.put(trait.getObservationVariableName(),trait); + } + for (PendingImportObject observation : this.observationByHash.values()) { + String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); + if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { + String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); + observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); + } + } + } + + // TODO: used by both workflows + public List commitNewPendingObservationsToBrAPIStore(ImportContext context, PendingData pendingData) { + // filter out observations with no 'value' so they will not be saved + List newObservations = ProcessorData.getNewObjects(this.observationByHash) + .stream() + .filter(obs -> !obs.getValue().isBlank()) + .collect(Collectors.toList()); + + updateObservationDependencyValues(program); + return brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); + + } + + // TODO: used by both workflows + public List commitUpdatedPendingObservationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedObservations = new ArrayList<>(); + Map mutatedObservationByDbId = ProcessorData + .getMutationsByObjectId(observationByHash, BrAPIObservation::getObservationDbId); + + for (Map.Entry entry : mutatedObservationByDbId.entrySet()) { + String id = entry.getKey(); + BrAPIObservation observation = entry.getValue(); + try { + if (observation == null) { + throw new Exception("Null observation"); + } + BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); + updatedObservations.add(updatedObs); + + if (updatedObs == null) { + throw new Exception("Null updated observation"); + } + + if (!Objects.equals(observation.getValue(), updatedObs.getValue()) + || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { + String message; + if (!Objects.equals(observation.getValue(), updatedObs.getValue())) { + message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); + } else { + message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); + } + throw new Exception(message); + } + + } catch (ApiException e) { + log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating observation: ", e); + throw new InternalServerException(e.getMessage(), e); + } + } + + return updatedObservations; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java new file mode 100644 index 000000000..dd8bc2856 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -0,0 +1,379 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import io.micronaut.context.annotation.Property; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; + +public class ObservationUnitService { + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public ObservationUnitService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + } + + /** + * Retrieves a list of BrAPI (Breeding API) observation units by their database IDs for a given set of experimental unit IDs and program. + * + * This method queries the BrAPIObservationUnitDAO to retrieve BrAPI observation units based on the provided experimental unit IDs and program. + * If the database IDs of the retrieved BrAPI observation units do not match the provided experimental unit IDs, an IllegalStateException is thrown. + * The exception includes information on the missing observation unit database IDs. + * + * @param expUnitIds a set of experimental unit IDs for which to retrieve BrAPI observation units + * @param program the program for which to retrieve BrAPI observation units + * @return a list of BrAPIObservationUnit objects corresponding to the provided experimental unit IDs + * @throws ApiException if an error occurs during the retrieval of observation units + * @throws IllegalStateException if the retrieved observation units do not match the provided experimental unit IDs + */ + public List getObservationUnitsByDbId(Set expUnitIds, Program program) throws ApiException, IllegalStateException { + List brapiUnits = null; + + // Retrieve reference Observation Units based on IDs + brapiUnits = brAPIObservationUnitDAO.getObservationUnitsById(expUnitIds, program); + + // If no BrAPI units are found, throw an IllegalStateException with an error message + if (expUnitIds.size() != brapiUnits.size()) { + Set missingIds = new HashSet<>(expUnitIds); + + // Calculate missing IDs based on retrieved BrAPI units + missingIds.removeAll(brapiUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toSet())); + + // Throw exception with missing IDs information + throw new IllegalStateException("Observation unit not found for unit dbid(s): " + String.join(COMMA_DELIMITER, missingIds)); + } + + return brapiUnits; + } + + /** + * Constructs a PendingImportObject of type BrAPIObservationUnit from a given BrAPIObservationUnit object. + * This function is responsible for constructing an import object that represents an observation unit for the Deltabreed system, + * using a provided BrAPIObservationUnit object from a BrAPI source. + * + * @param unit the BrAPIObservationUnit object to be used for constructing the PendingImportObject + * @return a PendingImportObject of type BrAPIObservationUnit representing the imported observation unit + * @throws IllegalStateException if the external reference for the observation unit does not exist + */ + public PendingImportObject constructPIOFromBrapiUnit(BrAPIObservationUnit unit) { + final PendingImportObject[] pio = new PendingImportObject[]{null}; + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + // Get external reference for the Observation Unit + Optional unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource); + unitXref.ifPresentOrElse( + xref -> { + pio[0] = new PendingImportObject(ImportObjectState.EXISTING, unit, UUID.fromString(xref.getReferenceId())); + }, + () -> { + + // but throw an error if no unit ID + throw new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID"); + } + ); + return pio[0]; + } + + /** + * Maps pending observation units by their reference IDs. + * This function takes a list of pending import objects representing BrAPI observation units + * and constructs a map where the key is the external reference ID of the observation unit + * and the value is the pending import object itself. + * + * @param pios List of pending import objects for BrAPI observation units + * @return A map of pending observation units keyed by their external reference ID + */ + public Map> mapPendingUnitById(List> pios) { + Map> pendingUnitById = new HashMap<>(); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + for (PendingImportObject pio : pios) { + + // Get external reference for the Observation Unit + Optional xref = Utilities.getExternalReference(pio.getBrAPIObject().getExternalReferences(), deltaBreedOUSource); + pendingUnitById.put(xref.get().getReferenceId(),pio); + } + + return pendingUnitById; + } + + /** + * This method takes a list of PendingImportObject objects and a Program object as input + * and maps the PendingImportObject objects by their observation unit name without the program scope. + * + * @param pios A list of PendingImportObject objects to be processed + * @param program The Program object representing the scope to be removed from observation unit names + * @return A Map> mapping observation unit names without the program scope to the corresponding PendingImportObject objects + */ + public Map> mapPendingUnitByNameNoScope(List> pios, + Program program) { + Map> pendingUnitByNameNoScope = new HashMap<>(); + + for (PendingImportObject pio : pios) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pio.getBrAPIObject().getStudyName(), + program.getKey() + ); + String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pio.getBrAPIObject().getObservationUnitName(), + program.getKey() + ); + pendingUnitByNameNoScope.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName), pio); + } + + return pendingUnitByNameNoScope; + } + + /** + * Collects missing Observation Unit IDs from a set of reference IDs and a list of existing Observation Units. + * + * This function takes a Set of reference IDs and a List of existing Observation Units, filters out the Observation Units + * that have external references matching a specific source, and returns a List of missing Observation Unit IDs that are + * present in the reference IDs but not found in the existing Observation Units. + * + * @param referenceIds The Set of reference IDs representing all possible Observation Unit IDs to match against. + * @param existingUnits The List of existing Observation Units to compare against the reference IDs. + * @return A List of Observation Unit IDs that are missing from the existing Observation Units but present in the reference IDs. + */ + public List collectMissingOUIds(Set referenceIds, List existingUnits) { + List missingIds = new ArrayList<>(referenceIds); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + Set fetchedIds = existingUnits.stream() + .filter(unit ->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).isPresent()) + .map(unit->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).get().getReferenceId()) + .collect(Collectors.toSet()); + missingIds.removeAll(fetchedIds); + + return missingIds; + } + + // TODO: used by expUnit workflow + public PendingImportObject fetchOrCreateObsUnitPIO(ImportContext importContext, + PendingData pendingData, + ExpUnitContext expUnitContext, + String envSeqValue) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { + PendingImportObject pio; + String key = createObservationUnitKey(importRow); + if (hasAllReferenceUnitIds) { + pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); + } else if (observationUnitByNameNoScope.containsKey(key)) { + pio = observationUnitByNameNoScope.get(key); + } else { + String germplasmName = ""; + if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { + germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) + .getBrAPIObject() + .getGermplasmName(); + } + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; + UUID trialID = trialPIO.getId(); + UUID datasetId = null; + if (commit) { + datasetId = UUID.fromString(trialPIO.getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + } + PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); + + // check for existing units if this is an existing study + if (studyPIO.getBrAPIObject().getStudyDbId() != null) { + List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); + List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); + if (matchingOU.isEmpty()) { + throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); + } else { + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); + } + } else { + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); + } + this.observationUnitByNameNoScope.put(key, pio); + } + return pio; + } + + // TODO: used by create workflow + public PendingImportObject fetchOrCreateObsUnitPIO(ImportContext importContext, + PendingData pendingData, + String envSeqValue) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { + PendingImportObject pio; + String key = createObservationUnitKey(importRow); + if (hasAllReferenceUnitIds) { + pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); + } else if (observationUnitByNameNoScope.containsKey(key)) { + pio = observationUnitByNameNoScope.get(key); + } else { + String germplasmName = ""; + if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { + germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) + .getBrAPIObject() + .getGermplasmName(); + } + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; + UUID trialID = trialPIO.getId(); + UUID datasetId = null; + if (commit) { + datasetId = UUID.fromString(trialPIO.getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + } + PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); + + // check for existing units if this is an existing study + if (studyPIO.getBrAPIObject().getStudyDbId() != null) { + List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); + List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); + if (matchingOU.isEmpty()) { + throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); + } else { + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); + } + } else { + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); + } + this.observationUnitByNameNoScope.put(key, pio); + } + return pio; + } + + // TODO: used by both workflows + public String createObservationUnitKey(ExperimentObservation importRow) { + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); + } + + // TODO: used by both workflows + public String createObservationUnitKey(String studyName, String obsUnitName) { + return studyName + obsUnitName; + } + + // TODO: used by create workflow + public void validateObservationUnits(ValidationErrors validationErrors, + Set uniqueStudyAndObsUnit, + int rowNum, + ExperimentObservation importRow) { + validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + + String key = createObservationUnitKey(importRow); + PendingImportObject ouPIO = observationUnitByNameNoScope.get(key); + if(ouPIO.getState() == ImportObjectState.NEW && StringUtils.isNotBlank(importRow.getObsUnitID())) { + addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "Could not find observation unit by ObsUnitDBID", validationErrors, rowNum); + } + + validateGeoCoordinates(validationErrors, rowNum, importRow); + } + + // TODO: used by both workflows + private void updateObsUnitDependencyValues(String programKey) { + + // update study DbIds + this.studyByNameNoScope.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(study -> updateStudyDbId(study, programKey)); + + // update germplasm DbIds + this.existingGermplasmByGID.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(this::updateGermplasmDbId); + } + + // TODO: used by both workflows + public List commitNewPendingObservationUnitsToBrAPIStore(ImportContext context, PendingData pendingData) { + List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); + updateObsUnitDependencyValues(program.getKey()); + List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); + + // set the DbId to the for each newly created Observation Unit + for (BrAPIObservationUnit createdObservationUnit : createdObservationUnits) { + // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope + String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getStudyName(), program.getKey()); + String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getObservationUnitName(), program.getKey()); + String createdObsUnit_key = createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); + this.observationUnitByNameNoScope.get(createdObsUnit_key) + .getBrAPIObject() + .setObservationUnitDbId(createdObservationUnit.getObservationUnitDbId()); + } + + return createdObservationUnits; + } + + // TODO: used by both workflows + public List commitUpdatedPendingObservationUnitToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedUnits = new ArrayList<>(); + + return updatedUnits; + } + + private void updateStudyDbId(BrAPIStudy study, String programKey) { + this.observationUnitByNameNoScope.values() + .stream() + .filter(obsUnit -> obsUnit.getBrAPIObject() + .getStudyName() + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), programKey))) + .forEach(obsUnit -> { + obsUnit.getBrAPIObject() + .setStudyDbId(study.getStudyDbId()); + obsUnit.getBrAPIObject() + .setTrialDbId(study.getTrialDbId()); + }); + } + + private void updateGermplasmDbId(BrAPIGermplasm germplasm) { + this.observationUnitByNameNoScope.values() + .stream() + .filter(obsUnit -> germplasm.getAccessionNumber() != null && + germplasm.getAccessionNumber().equals(obsUnit + .getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.GID).getAsString())) + .forEach(obsUnit -> obsUnit.getBrAPIObject() + .setGermplasmDbId(germplasm.getGermplasmDbId())); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java new file mode 100644 index 000000000..1671d77ee --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java @@ -0,0 +1,282 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.server.exceptions.InternalServerException; +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.OverwrittenData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.Utilities; +import tech.tablesaw.columns.Column; +import org.breedinginsight.dao.db.tables.pojos.TraitEntity; + +import javax.inject.Inject; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.stream.Collectors; + +public class ObservationVariableService { + private final OntologyService ontologyService; + + @Inject + public ObservationVariableService(OntologyService ontologyService) { + this.ontologyService = ontologyService; + } + + /** + * Fetches traits by name for the given set of variable names and program. + * + * This method fetches all stored traits for the specified program and filters them based on the set of variable names provided. + * It ensures that all requested observation variables are present and returns a list of matching traits. + * If any observation variables are missing, it throws an IllegalStateException with the missing variable names. + * + * @param varNames a set of variable names to fetch traits for + * @param program the program for which traits are fetched + * @return a list of traits filtered by the provided variable names + * @throws DoesNotExistException if the program or traits do not exist + * @throws IllegalStateException if any requested observation variables are missing + */ + public List fetchTraitsByName(Set varNames, Program program) throws DoesNotExistException, IllegalStateException { + List traits = null; + + // Fetch all stored traits for the program + List programTraits = ontologyService.getTraitsByProgramId(program.getId(), true); + + // Only keep traits that are in the set of names + List upperCaseVarNames = varNames.stream().map(String::toUpperCase).collect(Collectors.toList()); + traits = programTraits.stream().filter(e -> upperCaseVarNames.contains(e.getObservationVariableName().toUpperCase())).collect(Collectors.toList()); + + // If any requested observation variables are missing, throw an IllegalStateException + if (varNames.size() != traits.size()) { + Set missingVarNames = new HashSet<>(varNames); + missingVarNames.removeAll(traits.stream().map(TraitEntity::getObservationVariableName).collect(Collectors.toSet())); + throw new IllegalStateException("Observation variables not found for name(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingVarNames)); + } + + return traits; + } + + public Optional> validateMatchedTimestamps(Set observationVariableNames, + List> timestampCols) { + Optional> ve = Optional.empty(); + // Check that each ts column corresponds to a phenotype column + List valErrs = timestampCols.stream() + .filter(col -> !(observationVariableNames.contains(col.name().replaceFirst(ExpImportProcessConstants.TIMESTAMP_REGEX, StringUtils.EMPTY)))) + .map(col -> new ValidationError(col.name().toString(), String.format("Timestamp column %s lacks corresponding phenotype column", col.name().toString()), HttpStatus.UNPROCESSABLE_ENTITY)) + .collect(Collectors.toList()); + if (valErrs.size() > 0) { + ve = Optional.of(valErrs); + } + + return ve; + } + // TODO: used with expUnit workflow + public void validateObservations(PendingData pendingData, + int rowNum, + ImportContext importContext, + ExpUnitContext expUnitContext, + List> phenotypeCols, + CaseInsensitiveMap colVarMap) { + for (Column phenoCol : phenotypeCols) { + String importHash; + String importObsValue = phenoCol.getString(rowNum); + + importHash = getImportObservationHash( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(phenoCol), + pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() + ); + + validateObservation(importHash); + } + } + + // TODO: used with create workflow + public void validateObservations(PendingData pendingData, + int rowNum, + ImportContext importContext, + List> phenotypeCols, + CaseInsensitiveMap colVarMap) { + + + for (Column phenoCol : phenotypeCols) { + String importHash; + String importObsValue = phenoCol.getString(rowNum); + + importHash = getImportObservationHash(importRow, phenoCol.name()); + + validateObservation(importHash); + } + + } + + // TODO: used by both workflows + private void validateObservation(String importHash) { + + + String importObsValue = phenoCol.getString(rowNum); + + + // error if import observation data already exists and user has not selected to overwrite + if (commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && + this.existingObsByObsHash.containsKey(importHash) && + StringUtils.isNotBlank(phenoCol.getString(rowNum)) && + !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { + addRowError( + phenoCol.name(), + String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), + validationErrors, rowNum + ); + + // preview case where observation has already been committed and the import row ObsVar data differs from what + // had been saved prior to import + } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { + + // add a change log entry when updating the value of an observation + if (commit) { + BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; + String prior = ""; + if (isValueMatched(importHash, importObsValue)) { + prior.concat(existingObsByObsHash.get(importHash).getValue()); + } + if (timeStampColByPheno.containsKey(phenoCol.name()) && isTimestampMatched(importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { + prior = prior.isEmpty() ? prior : prior.concat(" "); + prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); + } + ChangeLogEntry change = new ChangeLogEntry(prior, + reason, + user.getId(), + timestamp + ); + + // create the changelog field in additional info if it does not already exist + OverwrittenData.createChangeLog(pendingObservation); + + // add a new entry to the changelog + pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); + } + + // preview case where observation has already been committed and import ObsVar data is the + // same as has been committed prior to import + } else if (isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { + BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); + existingObs.setObservationVariableName(phenoCol.name()); + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + observationByHash.get(importHash).setBrAPIObject(existingObs); + + // preview case where observation has already been committed and import ObsVar data is empty prior to import + } else if (!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + } else { + validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); + + //Timestamp validation + if (timeStampColByPheno.containsKey(phenoCol.name())) { + Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); + validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); + } + } + + } + + // TODO: used by both workflows + private void updateObservationDependencyValues(Program program) { + String programKey = program.getKey(); + + // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds + this.observationUnitByNameNoScope.values().stream() + .map(PendingImportObject::getBrAPIObject) + .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); + + // Update ObservationVariable DbIds + List traits = getTraitList(program); + CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); + for ( Trait trait: traits) { + traitMap.put(trait.getObservationVariableName(),trait); + } + for (PendingImportObject observation : this.observationByHash.values()) { + String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); + if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { + String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); + observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); + } + } + } + + // TODO: used by both workflows + public List commitNewPendingObservationsToBrAPIStore(ImportContext context, PendingData pendingData) { + // filter out observations with no 'value' so they will not be saved + List newObservations = ProcessorData.getNewObjects(this.observationByHash) + .stream() + .filter(obs -> !obs.getValue().isBlank()) + .collect(Collectors.toList()); + + updateObservationDependencyValues(program); + return brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); + + } + + // TODO: used by both workflows + public List commitUpdatedPendingObservationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedObservations = new ArrayList<>(); + Map mutatedObservationByDbId = ProcessorData + .getMutationsByObjectId(observationByHash, BrAPIObservation::getObservationDbId); + + for (Map.Entry entry : mutatedObservationByDbId.entrySet()) { + String id = entry.getKey(); + BrAPIObservation observation = entry.getValue(); + try { + if (observation == null) { + throw new Exception("Null observation"); + } + BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); + updatedObservations.add(updatedObs); + + if (updatedObs == null) { + throw new Exception("Null updated observation"); + } + + if (!Objects.equals(observation.getValue(), updatedObs.getValue()) + || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { + String message; + if (!Objects.equals(observation.getValue(), updatedObs.getValue())) { + message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); + } else { + message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); + } + throw new Exception(message); + } + + } catch (ApiException e) { + log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating observation: ", e); + throw new InternalServerException(e.getMessage(), e); + } + } + + return updatedObservations; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StatisticsService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StatisticsService.java new file mode 100644 index 000000000..8b9844cdb --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StatisticsService.java @@ -0,0 +1,92 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; +import org.breedinginsight.services.exceptions.ValidatorException; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +public class StatisticsService { + // TODO: used by both workflows + public Map generateStatisticsMap(List importRows) { + // Data for stats. + HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names + HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's + HashSet gidCounter = new HashSet<>(); // set of unique GID's + + for (BrAPIImport row : importRows) { + ExperimentObservation importRow = (ExperimentObservation) row; + // Collect date for stats. + addIfNotNull(environmentNameCounter, importRow.getEnv()); + addIfNotNull(obsUnitsIDCounter, createObservationUnitKey(importRow)); + addIfNotNull(gidCounter, importRow.getGid()); + } + + int numNewObservations = Math.toIntExact( + observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + int numExistingObservations = Math.toIntExact( + this.observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.EXISTING && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + int numMutatedObservations = Math.toIntExact( + this.observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() + .newObjectCount(environmentNameCounter.size()) + .build(); + ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() + .newObjectCount(obsUnitsIDCounter.size()) + .build(); + ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() + .newObjectCount(gidCounter.size()) + .build(); + ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder() + .newObjectCount(numNewObservations) + .build(); + ImportPreviewStatistics existingObservationStats = ImportPreviewStatistics.builder() + .newObjectCount(numExistingObservations) + .build(); + ImportPreviewStatistics mutatedObservationStats = ImportPreviewStatistics.builder() + .newObjectCount(numMutatedObservations) + .build(); + + return Map.of( + "Environments", environmentStats, + "Observation_Units", obdUnitStats, + "GIDs", gidStats, + "Observations", observationStats, + "Existing_Observations", existingObservationStats, + "Mutated_Observations", mutatedObservationStats + ); + } + + // TODO: used by both workflows + public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { + // TODO + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java new file mode 100644 index 000000000..f6679e947 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -0,0 +1,397 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import io.micronaut.context.annotation.Property; +import io.reactivex.functions.Function; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPISeason; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.math.BigInteger; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; + +@Singleton +@Slf4j +public class StudyService { + private final Map seasonDbIdToYearCache = new HashMap<>(); + private final BrAPISeasonDAO brAPISeasonDAO; + private final BrAPIStudyDAO brAPIStudyDAO; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public StudyService(BrAPISeasonDAO brAPISeasonDAO, + BrAPIStudyDAO brAPIStudyDAO) { + this.brAPISeasonDAO = brAPISeasonDAO; + this.brAPIStudyDAO = brAPIStudyDAO; + } + + // TODO: used by both workflows + public PendingImportObject processAndCacheStudy( + BrAPIStudy existingStudy, + Program program, + Function getterFunction, + Map> studyMap) throws Exception { + PendingImportObject pendingStudy; + BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); + // map season dbid to year + String seasonDbId = existingStudy.getSeasons().get(0); // It is assumed that the study has only one season + if(StringUtils.isNotBlank(seasonDbId)) { + String seasonYear = seasonDbIdToYear(seasonDbId, program.getId()); + existingStudy.setSeasons(Collections.singletonList(seasonYear)); + } + pendingStudy = new PendingImportObject<>( + ImportObjectState.EXISTING, + (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), + UUID.fromString(xref.getReferenceId()) + ); + studyMap.put( + Utilities.removeProgramKeyAndUnknownAdditionalData(getterFunction.apply(existingStudy), program.getKey()), + pendingStudy + ); + return pendingStudy; + } + + /** + * Constructs a PendingImportObject containing a BrAPIStudy object based on the provided BrAPIStudy and Program. + * This function retrieves the external reference for the study and maps the season dbid to the corresponding year. + * + * @param brAPIStudy The BrAPIStudy object to construct the PendingImportObject from. + * @param program The Program object associated with the study. + * @return A PendingImportObject containing the formatted BrAPIStudy object. + * @throws IllegalStateException If the external reference for the study is not found. + */ + public PendingImportObject constructPIOFromBrapiStudy(BrAPIStudy brAPIStudy, Program program) { + // Retrieve external reference for the study + BrAPIExternalReference xref = Utilities.getExternalReference(brAPIStudy.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) + .orElseThrow(() -> new IllegalStateException("External references weren't found for study (dbid): " + brAPIStudy.getStudyDbId())); + + // Map season dbid to year + String seasonDbId = brAPIStudy.getSeasons().get(0); // It is assumed that the study has only one season + if(StringUtils.isNotBlank(seasonDbId)) { + String seasonYear = seasonDbIdToYear(seasonDbId, program.getId()); + brAPIStudy.setSeasons(Collections.singletonList(seasonYear)); + } + + // Create and return a PendingImportObject for the BrAPIStudy + return new PendingImportObject<>( + ImportObjectState.EXISTING, + (BrAPIStudy) Utilities.formatBrapiObjForDisplay(brAPIStudy, BrAPIStudy.class, program), + UUID.fromString(xref.getReferenceId()) + ); + } + + // TODO: used by both workflows + public String seasonDbIdToYear(String seasonDbId, UUID programId) { + String year = null; + // TODO: add season objects to redis cache then just extract year from those + // removing this for now here + //if (this.seasonDbIdToYearCache.containsKey(seasonDbId)) { // get it from cache if possible + // year = this.seasonDbIdToYearCache.get(seasonDbId); + //} else { + year = seasonDbIdToYearFromDatabase(seasonDbId, programId); + // this.seasonDbIdToYearCache.put(seasonDbId, year); + //} + return year; + } + + // TODO: used by both workflows + private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { + BrAPISeason season = null; + try { + season = this.brAPISeasonDAO.getSeasonById(seasonDbId, programId); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + } + Integer yearInt = (season == null) ? null : season.getYear(); + return (yearInt == null) ? "" : yearInt.toString(); + } + + /** + * Fetches a list of BrAPI studies by their study database IDs for a given program. + * + * This method queries the BrAPIStudyDAO to retrieve studies based on the provided study database IDs and the program. + * It ensures that all requested study database IDs are found in the result set, throwing an IllegalStateException if any are missing. + * + * @param studyDbIds a Set of Strings representing the study database IDs to fetch + * @param program the Program object representing the program context in which to fetch studies + * @return a List of BrAPIStudy objects matching the provided study database IDs + * + * @throws ApiException if there is an issue fetching the studies + * @throws IllegalStateException if any requested study database IDs are not found in the result set + */ + public List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { + List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); + if (studies.size() != studyDbIds.size()) { + List missingIds = new ArrayList<>(studyDbIds); + missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); + throw new IllegalStateException( + "Study not found for studyDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + return studies; + } + + /** + * Fetch BrAPI studies by their database identifiers for a given program. + * + * This method retrieves a list of BrAPI studies based on the provided set of study database identifiers + * and a specified program. It utilizes the BrAPIStudyDAO to fetch studies from the database. + * + * @param studyDbIds A set of study database identifiers for filtering the studies. + * @param program The program related to the studies. + * @return A list of BrAPIStudy objects representing the fetched studies. + * @throws ApiException If there are issues in retrieving studies or if any study database identifier is missing. + */ + public List fetchBrapiStudiesByDbId(Set studyDbIds, Program program) throws ApiException { + List brapiStudies = null; // Initializing the study object + brapiStudies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); // Retrieving studies from the database + + // If no studies are found, throw an IllegalStateException with an error message + if (studyDbIds.size() != brapiStudies.size()) { + Set missingIds = new HashSet<>(studyDbIds); + missingIds.removeAll(brapiStudies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toSet())); + throw new IllegalStateException("Study not found for location dbid(s): " + String.join(COMMA_DELIMITER, missingIds)); + } + + return brapiStudies; + } + + /** + * Retrieves the study database ID belonging to a pending unit in BrAPI format. + * + * This method takes a PendingImportObject containing a BrAPIObservationUnit + * object and returns the study database ID associated with the unit, if it exists. + * + * @param pio The PendingImportObject containing the BrAPIObservationUnit object for which the study database ID is to be retrieved. + * @return The study database ID belonging to the pending unit, or null if the unit does not exist or if the study database ID is not set. + */ + public String getStudyDbIdBelongingToPendingUnit(PendingImportObject pio) { + String studyDbId = null; + + // Check if the BrAPI object in the PendingImportObject is not null + if (pio.getBrAPIObject() != null) { + // Retrieve the study database ID from the BrAPIObservationUnit object + studyDbId = pio.getBrAPIObject().getStudyDbId(); + } + + return studyDbId; + } + + // TODO: used by both workflows + private void initializeStudiesForExistingObservationUnits( + Program program, + Map> studyByName + ) throws Exception { + Set studyDbIds = observationUnitByNameNoScope.values() + .stream() + .map(pio -> pio.getBrAPIObject() + .getStudyDbId()) + .collect(Collectors.toSet()); + + List studies = fetchStudiesByDbId(studyDbIds, program); + for (BrAPIStudy study : studies) { + processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); + } + } + + // TODO: used by expunit workflow + public Map> mapPendingStudyByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> studyByName, + Map> studyByOUId, + Program program + ) { + if (unit.getStudyName() != null) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); + studyByOUId.put(unitId, studyByName.get(studyName)); + } else { + throw new IllegalStateException("Observation unit missing study name: " + unitId); + } + + return studyByOUId; + } + + // TODO: used by expunit workflow + private PendingImportObject fetchOrCreateStudyPIO( + ImportContext importContext, + ExpUnitContext expUnitContext, + String expSequenceValue, + Supplier envNextVal + ) throws UnprocessableEntityException { + PendingImportObject pio; + if (hasAllReferenceUnitIds) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), + program.getKey() + ); + pio = studyByNameNoScope.get(studyName); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { + pio = studyByNameNoScope.get(importRow.getEnv()); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else { + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + UUID trialID = trialPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); + newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies + + // It is assumed that the study has only one season, And that the Years and not + // the dbId's are stored in getSeason() list. + String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season + if (commit) { + if(StringUtils.isNotBlank(year)) { + String seasonID = this.yearToSeasonDbId(year, program.getId()); + newStudy.setSeasons(Collections.singletonList(seasonID)); + } + } else { + addYearToStudyAdditionalInfo(program, newStudy, year); + } + + pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); + this.studyByNameNoScope.put(importRow.getEnv(), pio); + } + return pio; + } + + // TODO: used by create workflow + private PendingImportObject fetchOrCreateStudyPIO( + ImportContext importContext, + String expSequenceValue, + Supplier envNextVal + ) throws UnprocessableEntityException { + PendingImportObject pio; + if (hasAllReferenceUnitIds) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), + program.getKey() + ); + pio = studyByNameNoScope.get(studyName); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { + pio = studyByNameNoScope.get(importRow.getEnv()); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else { + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + UUID trialID = trialPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); + newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies + + // It is assumed that the study has only one season, And that the Years and not + // the dbId's are stored in getSeason() list. + String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season + if (commit) { + if(StringUtils.isNotBlank(year)) { + String seasonID = this.yearToSeasonDbId(year, program.getId()); + newStudy.setSeasons(Collections.singletonList(seasonID)); + } + } else { + addYearToStudyAdditionalInfo(program, newStudy, year); + } + + pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); + this.studyByNameNoScope.put(importRow.getEnv(), pio); + } + return pio; + } + + private void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { + // update location DbIds in studies for all distinct locations + mappedBrAPIImport.values() + .stream() + .map(PendingImport::getLocation) + .forEach(this::updateStudyLocationDbId); + + // update trial DbIds in studies for all distinct trials + this.trialByNameNoScope.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(trial -> this.updateTrialDbId(trial, programKey)); + } + + private void updateStudyLocationDbId(PendingImportObject location) { + this.studyByNameNoScope.values() + .stream() + .filter(study -> location.getId().toString() + .equals(study.getBrAPIObject() + .getLocationDbId())) + .forEach(study -> study.getBrAPIObject() + .setLocationDbId(location.getBrAPIObject().getLocationDbId())); + } + + private void updateTrialDbId(BrAPITrial trial, String programKey) { + this.studyByNameNoScope.values() + .stream() + .filter(study -> study.getBrAPIObject() + .getTrialName() + .equals(Utilities.removeProgramKey(trial.getTrialName(), programKey))) + .forEach(study -> study.getBrAPIObject() + .setTrialDbId(trial.getTrialDbId())); + } + + // TODO: used by both workflows + public List commitNewPendingStudiessToBrAPIStore(ImportContext context, PendingData pendingData) { + List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); + updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); + List createdStudies = brAPIStudyDAO.createBrAPIStudies(newStudies, program.getId(), upload); + + // set the DbId to the for each newly created study + for (BrAPIStudy createdStudy : createdStudies) { + String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData(createdStudy.getStudyName(), program.getKey()); + this.studyByNameNoScope.get(createdStudy_name_no_key) + .getBrAPIObject() + .setStudyDbId(createdStudy.getStudyDbId()); + } + + return createdStudies; + } + + // TODO: used by both workflows + public List commitUpdatedPendingStudiesToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedStudies = new ArrayList<>(); + + return updatedStudies; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java new file mode 100644 index 000000000..b82c5169d --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -0,0 +1,450 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; + +@Singleton +@Slf4j +public class TrialService { + private final BrAPITrialDAO brAPITrialDAO; + + private final StudyService studyService; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public TrialService(BrAPITrialDAO brAPITrialDAO, + StudyService studyService) { + this.brAPITrialDAO = brAPITrialDAO; + this.studyService = studyService; + } + /** + * Module: BrAPITrialService + * Description: This module contains methods for retrieving BrAPI trials based on trial database IDs and programs. + * brAPITrialDAO: Data Access Object for interacting with BrAPI trials in the database. + * fetchBrapiTrialsByDbId: Method to fetch BrAPI trials based on provided trial database IDs and program. + */ + + /** + * Retrieves the TrialDbId belonging to a pending unit based on the provided BrAPI observation unit and program. + * If the TrialDbId is directly assigned to the unit, it is returned. Otherwise, the TrialDbId + * assigned to the study belonging to the unit is retrieved. + * + * @param pendingUnit The pending import object representing the BrAPI observation unit. + * @param program The program associated with the pending import. + * @return The TrialDbId belonging to the pending unit. + * @throws IllegalStateException if TrialDbId and StudyDbId are not set for an existing ObservationUnit. + * @throws InternalServerException if there is an internal server error while fetching the TrialDbId. + */ + public String getTrialDbIdBelongingToPendingUnit(PendingImportObject pendingUnit, + Program program) throws IllegalStateException, InternalServerException { + String trialDbId = null; + + BrAPIObservationUnit brapiUnit = pendingUnit.getBrAPIObject(); + if (StringUtils.isBlank(brapiUnit.getTrialDbId()) && StringUtils.isBlank(brapiUnit.getStudyDbId())) { + throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); + } + + // get the trial directly assigned to the unit + if (StringUtils.isNotBlank(brapiUnit.getTrialDbId())) { + trialDbId = brapiUnit.getTrialDbId(); + } else { + + // or get the trial directly assigned to the study belonging to the unit + String studyDbId = brapiUnit.getStudyDbId(); + try { + trialDbId = getTrialDbIdBelongingToStudy(studyDbId, program); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + return trialDbId; + } + + /** + * Fetches BrAPI trials belonging to the specified trial database IDs and program. + * + * This method retrieves a list of BrAPI trials that are associated with the provided set of trial database IDs + * within the context of the given program. It ensures that all specified trial IDs are found in the result + * to maintain data integrity. + * + * @param trialDbIds A set of trial database IDs indicating the trials to fetch. + * @param program The program object representing the program to which the trials belong. + * @return A list of BrAPITrial objects representing the fetched trials. + * @throws IllegalStateException If not all specified trial database IDs are found in the fetched trials. + * @throws InternalServerException If an error occurs during the API call to fetch the trials. + */ + public List fetchBrapiTrialsBelongingToUnits(Set trialDbIds, Program program) { + try { + List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); + if (trials.size() != trialDbIds.size()) { + List missingIds = new ArrayList<>(trialDbIds); + missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); + throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + + return trials; + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + /** + * Retrieves a list of BrAPI trials based on a set of trial database IDs and a specified program. + * + * @param trialDbIds a set of trial database IDs used to retrieve the BrAPI trials + * @param program the program associated with the trials + * @return a list of BrAPITrial objects that match the provided trial database IDs and program + * @throws InternalServerException if there is an internal server error during the retrieval process + * @throws ApiException if there is an exception while fetching the trials + */ + public List fetchBrapiTrialsByDbId(Set trialDbIds, Program program) throws InternalServerException, ApiException { + // Initialize the list of BrAPI trials + List brapiTrials = null; + + // Retrieve the trials from the DAO based on the provided trial database IDs and program + brapiTrials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); + + // Check if all requested trials were found + if (trialDbIds.size() != brapiTrials.size()) { + // Identify the missing trial database IDs + Set missingIds = new HashSet<>(trialDbIds); + missingIds.removeAll(brapiTrials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toSet())); + + // Throw an exception with the list of missing trial database IDs + throw new IllegalStateException("Trial not found for trial dbid(s): " + String.join(COMMA_DELIMITER, missingIds)); + } + + // Return the list of retrieved BrAPI trials + return brapiTrials; + } + + + // TODO: also used in other workflow + + /** + * Initializes trials for existing observation units. + * + * @param program The program object. + * @param observationUnitByNameNoScope A map containing observation units by name (without scope). + * @param trialByName A map containing trials by name. (will be modified in place) + * + */ + public void initializeTrialsForExistingObservationUnits(ImportContext importContext, + PendingData pendingData) { + if(pendingData.getObservationUnitByNameNoScope().size() > 0) { + Set trialDbIds = new HashSet<>(); + Set studyDbIds = new HashSet<>(); + + pendingData.getObservationUnitByNameNoScope().values() + .forEach(pio -> { + BrAPIObservationUnit existingOu = pio.getBrAPIObject(); + if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { + throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); + } + + if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { + trialDbIds.add(existingOu.getTrialDbId()); + } else { + studyDbIds.add(existingOu.getStudyDbId()); + } + }); + + //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId + if(!studyDbIds.isEmpty()) { + try { + trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, importContext.getProgram())); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + try { + List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, importContext.getProgram()); + if (trials.size() != trialDbIds.size()) { + List missingIds = new ArrayList<>(trialDbIds); + missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); + throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + + trials.forEach(trial -> processAndCacheTrial(trial, importContext.getProgram(), pendingData.getTrialByNameNoScope())); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + } + + /** + * Fetches trial DbIds for the given study DbIds by using the BrAPI studies API. + * + * @param studyDbIds The set of study DbIds for which to fetch trial DbIds. + * @param program The program associated with the studies. + * @return A set of trial DbIds corresponding to the provided study DbIds. + * @throws ApiException If there was an error while fetching the studies or if a study does not have a trial DbId. + * @throws IllegalStateException If the trial DbId is not set for an existing study. + */ + private Set fetchTrialDbidsForStudies(Set studyDbIds, Program program) throws ApiException { + Set trialDbIds = new HashSet<>(); + List studies = studyService.fetchStudiesByDbId(studyDbIds, program); + studies.forEach(study -> { + if (StringUtils.isBlank(study.getTrialDbId())) { + throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); + } + trialDbIds.add(study.getTrialDbId()); + }); + + return trialDbIds; + } + + /** + * Retrieves the trialDbId that belongs to a specific study with the given studyDbId and program. + * + * @param studyDbId The unique identifier for the study in the system. + * @param program The program object associated with the study. + * @return The trialDbId associated with the study. + * @throws ApiException if the study with the provided studyDbId is not found in the BrAPI service. + * @throws IllegalStateException if the trialDbId is not set for the existing study. + */ + private String getTrialDbIdBelongingToStudy(String studyDbId, Program program) throws ApiException { + String trialDbId = null; + List studies = studyService.fetchStudiesByDbId(Set.of(studyDbId), program); + if (studies.size() == 0) { + throw new ApiException("Study not found in BrAPI service: " + studyDbId); + } + BrAPIStudy study = studies.get(0); + if (StringUtils.isBlank(study.getTrialDbId())) { + throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); + } + trialDbId = study.getTrialDbId(); + + return trialDbId; + } + + /** + * This method processes an existing trial, retrieves the experiment ID from the trial's external references, + * and caches the trial with the corresponding experiment ID in a map. + * + * @param existingTrial The existing BrAPITrial object to be processed and cached. + * @param program The Program object associated with the trial. + * @param trialByNameNoScope The map to cache the trial by its name without program scope. (will be modified in place) + * + * @throws InternalServerException + */ + private void processAndCacheTrial( + BrAPITrial existingTrial, + Program program, + Map> trialByNameNoScope) { + + //get TrialId from existingTrial + BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) + .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); + UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); + + trialByNameNoScope.put( + Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), + new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); + } + + /** + * Constructs a PendingImportObject containing a BrAPITrial object based on the input BrAPITrial. + * + * This function takes a BrAPITrial object as input and constructs a PendingImportObject which + * encapsulates the trial along with its associated experiment ID. The experiment ID is retrieved + * from the external references of the trial object using utility method getExternalReference. + * If the experiment ID is not found in the external references, an InternalServerException is thrown. + * + * @param trial the BrAPITrial object for which the PendingImportObject is to be constructed + * @return a PendingImportObject containing the BrAPITrial object and its associated experiment ID + * @throws InternalServerException if the experiment ID is not found in the external references of the trial + */ + public PendingImportObject constructPIOFromBrapiTrial(BrAPITrial trial) throws InternalServerException { + PendingImportObject pio = null; + BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(trial.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) + .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); + UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); + pio = new PendingImportObject<>(ImportObjectState.EXISTING, trial, experimentId); + return pio; + } + + /** + * Initializes trials by name without scope for the given program. + * + * @param program the program to initialize trials for + * @param observationUnitByNameNoScope a map of observation units by name without scope + * @param experimentImportRows a list of experiment observation rows + * @return a map of trials by name with pending import objects + * + * @throws InternalServerException + */ + private Map> initializeTrialByNameNoScope(Program program, Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> trialByName = new HashMap<>(); + + initializeTrialsForExistingObservationUnits(program, observationUnitByNameNoScope, trialByName); + + List uniqueTrialNames = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID())) + .map(ExperimentObservation::getExpTitle) + .distinct() + .collect(Collectors.toList()); + try { + brAPITrialDAO.getTrialsByName(uniqueTrialNames, program).forEach(existingTrial -> + processAndCacheTrial(existingTrial, program, trialByName) + ); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + return trialByName; + } + + private void initializeTrialsForExistingObservationUnits(Program program, Map> observationUnitByNameNoScope, Map> trialByName) { + } + + // TODO: used by expunit workflow + public Map> mapPendingTrialByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> trialByName, + Map> studyByName, + Map> trialByOUId, + Program program + ) { + String trialName; + if (unit.getTrialName() != null) { + trialName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getTrialName(), program.getKey()); + } else if (unit.getStudyName() != null) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); + trialName = Utilities.removeProgramKeyAndUnknownAdditionalData( + studyByName.get(studyName).getBrAPIObject().getTrialName(), + program.getKey() + ); + } else { + throw new IllegalStateException("Observation unit missing trial name and study name: " + unitId); + } + trialByOUId.put(unitId, trialByName.get(trialName)); + + return trialByOUId; + } + + // TODO: overloaded method used by expunit workflow + public PendingImportObject fetchOrCreateTrialPIO( + ImportContext importContext, + PendingData pendingData, + ExpUnitContext expUnitContext + ) throws UnprocessableEntityException { + PendingImportObject trialPio; + + + + trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); + + + return trialPio; + } + + // TODO: overloaded method used by create workflow + public PendingImportObject fetchOrCreateTrialPIO( + ImportContext importContext, + PendingData pendingData + ) throws UnprocessableEntityException { + PendingImportObject trialPio; + + + if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { + PendingImportObject envPio; + trialPio = trialByNameNoScope.get(importRow.getExpTitle()); + envPio = studyByNameNoScope.get(importRow.getEnv()); + + // creating new units for existing experiments and environments is not possible + if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && + (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ + throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); + } + } else if (!trialByNameNoScope.isEmpty()) { + throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); + } else { + UUID id = UUID.randomUUID(); + String expSeqValue = null; + if (commit) { + expSeqValue = expNextVal.get().toString(); + } + BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); + trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); + trialByNameNoScope.put(importRow.getExpTitle(), trialPio); + } + + return trialPio; + } + + // TODO: used by both workflows + public List commitNewPendingTrialsToBrAPIStore(ImportContext context, PendingData pendingData) { + List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); + List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newTrials, program.getId(), upload)); + // set the DbId to the for each newly created trial + for (BrAPITrial createdTrial : createdTrials) { + String createdTrialName = Utilities.removeProgramKey(createdTrial.getTrialName(), program.getKey()); + this.trialByNameNoScope.get(createdTrialName) + .getBrAPIObject() + .setTrialDbId(createdTrial.getTrialDbId()); + } + return createdTrials; + } + + public List commitUpdatedPendingTrialsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedTrials = new ArrayList<>(); + Map mutatedTrialsById = ProcessorData + .getMutationsByObjectId(trialByNameNoScope, BrAPITrial::getTrialDbId); + for (Map.Entry entry : mutatedTrialsById.entrySet()) { + String id = entry.getKey(); + BrAPITrial trial = entry.getValue(); + try { + updatedTrials.add(brapiTrialDAO.updateBrAPITrial(id, trial, program.getId())); + } catch (ApiException e) { + log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating dataset observation variables: ", e); + throw new InternalServerException(e.getMessage(), e); + } + } + return updatedTrials; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java new file mode 100644 index 000000000..dd7bc0af2 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java @@ -0,0 +1,312 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.apache.commons.lang3.StringUtils; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.utilities.Utilities; +import tech.tablesaw.columns.Column; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class ValidateService { + // TODO: used by expUnit workflow + public void prepareDataForValidation(ImportContext importContext, + ExpUnitContext expUnitContext, + List> phenotypeCols) { + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); + List> observations = mappedImportRow.getObservations(); + String observationHash; + if (hasAllReferenceUnitIds) { + String refOUId = importRow.getObsUnitID(); + mappedImportRow.setTrial(pendingTrialByOUId.get(refOUId)); + mappedImportRow.setLocation(pendingLocationByOUId.get(refOUId)); + mappedImportRow.setStudy(pendingStudyByOUId.get(refOUId)); + mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(refOUId)); + mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(refOUId)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + observationHash = getObservationHash( + pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + + pendingObsUnitByOUId.get(refOUId).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(column), + pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + ); + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(observationHash)); + } + + } else { + mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); + mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); + mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); + mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); + mappedImportRow.setGermplasm(getGidPIO(importRow)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); + } + } + + mappedBrAPIImport.put(rowNum, mappedImportRow); + } + } + + // TODO: used by create workflow + public void prepareDataForValidation(ImportContext importContext, + List> phenotypeCols) { + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); + List> observations = mappedImportRow.getObservations(); + String observationHash; + if (hasAllReferenceUnitIds) { + String refOUId = importRow.getObsUnitID(); + mappedImportRow.setTrial(pendingTrialByOUId.get(refOUId)); + mappedImportRow.setLocation(pendingLocationByOUId.get(refOUId)); + mappedImportRow.setStudy(pendingStudyByOUId.get(refOUId)); + mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(refOUId)); + mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(refOUId)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + observationHash = getObservationHash( + pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + + pendingObsUnitByOUId.get(refOUId).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(column), + pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + ); + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(observationHash)); + } + + } else { + mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); + mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); + mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); + mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); + mappedImportRow.setGermplasm(getGidPIO(importRow)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); + } + } + + mappedBrAPIImport.put(rowNum, mappedImportRow); + } + } + + // TODO: used by expUnit workflow + public void validateFields(ImportContext importContext, + PendingData pendingData, + ExpUnitContext expUnitContext, + List referencedTraits, Program program, + List> phenotypeCols) { + //fetching any existing observations for any OUs in the import + CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); + for ( Trait trait: referencedTraits) { + colVarMap.put(trait.getObservationVariableName(),trait); + } + Set uniqueStudyAndObsUnit = new HashSet<>(); + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); + if (hasAllReferenceUnitIds) { + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); + } else { + if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. + validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); + } + validateTestOrCheck(importRow, validationErrors, rowNum); + validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); + validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); + } + } + } + + // TODO: used by create workflow + public void validateFields(ImportContext importContext, + PendingData pendingData, + List referencedTraits, Program program, + List> phenotypeCols) { + //fetching any existing observations for any OUs in the import + CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); + for ( Trait trait: referencedTraits) { + colVarMap.put(trait.getObservationVariableName(),trait); + } + Set uniqueStudyAndObsUnit = new HashSet<>(); + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); + if (hasAllReferenceUnitIds) { + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); + } else { + if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. + validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); + } + validateTestOrCheck(importRow, validationErrors, rowNum); + validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); + validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); + } + } + } + + // TODO: used by create workflow + private void validateTestOrCheck(ExperimentObservation importRow, ValidationErrors validationErrors, int rowNum) { + String testOrCheck = importRow.getTestOrCheck(); + if ( ! ( testOrCheck==null || testOrCheck.isBlank() + || "C".equalsIgnoreCase(testOrCheck) || "CHECK".equalsIgnoreCase(testOrCheck) + || "T".equalsIgnoreCase(testOrCheck) || "TEST".equalsIgnoreCase(testOrCheck) ) + ){ + addRowError(ExperimentObservation.Columns.TEST_CHECK, String.format("Invalid value (%s)", testOrCheck), validationErrors, rowNum) ; + } + } + + // TODO: used by create workflow + private void validateConditionallyRequired(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, Program program, boolean commit) { + ImportObjectState expState = this.trialByNameNoScope.get(importRow.getExpTitle()) + .getState(); + ImportObjectState envState = this.studyByNameNoScope.get(importRow.getEnv()).getState(); + + String errorMessage = BLANK_FIELD_EXPERIMENT; + if (expState == ImportObjectState.EXISTING && envState == ImportObjectState.NEW) { + errorMessage = BLANK_FIELD_ENV; + } else if(expState == ImportObjectState.EXISTING && envState == ImportObjectState.EXISTING) { + errorMessage = BLANK_FIELD_OBS; + } + + if(expState == ImportObjectState.NEW || envState == ImportObjectState.NEW) { + validateRequiredCell(importRow.getGid(), ExperimentObservation.Columns.GERMPLASM_GID, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpTitle(), ExperimentObservation.Columns.EXP_TITLE,errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpUnit(), ExperimentObservation.Columns.EXP_UNIT, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpType(), ExperimentObservation.Columns.EXP_TYPE, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getEnv(), ExperimentObservation.Columns.ENV, errorMessage, validationErrors, rowNum); + if(validateRequiredCell(importRow.getEnvLocation(), ExperimentObservation.Columns.ENV_LOCATION, errorMessage, validationErrors, rowNum)) { + if(!Utilities.removeProgramKeyAndUnknownAdditionalData(this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getLocationName(), program.getKey()).equals(importRow.getEnvLocation())) { + addRowError(ExperimentObservation.Columns.ENV_LOCATION, ENV_LOCATION_MISMATCH, validationErrors, rowNum); + } + } + if(validateRequiredCell(importRow.getEnvYear(), ExperimentObservation.Columns.ENV_YEAR, errorMessage, validationErrors, rowNum)) { + String studyYear = StringUtils.defaultString( this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getSeasons().get(0) ); + String rowYear = importRow.getEnvYear(); + if(commit) { + rowYear = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); + } + if(StringUtils.isNotBlank(studyYear) && !studyYear.equals(rowYear)) { + addRowError(ExperimentObservation.Columns.ENV_YEAR, ENV_YEAR_MISMATCH, validationErrors, rowNum); + } + } + validateRequiredCell(importRow.getExpUnitId(), ExperimentObservation.Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpReplicateNo(), ExperimentObservation.Columns.REP_NUM, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpBlockNo(), ExperimentObservation.Columns.BLOCK_NUM, errorMessage, validationErrors, rowNum); + + if(StringUtils.isNotBlank(importRow.getObsUnitID())) { + addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); + } + } else { + //Check if existing environment. If so, ObsUnitId must be assigned + validateRequiredCell( + importRow.getObsUnitID(), + ExperimentObservation.Columns.OBS_UNIT_ID, + MISSING_OBS_UNIT_ID_ERROR, + validationErrors, + rowNum + ); + } + } + + // TODO: used by create workflow + public void validateGeoCoordinates(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow) { + + String lat = importRow.getLatitude(); + String lon = importRow.getLongitude(); + String elevation = importRow.getElevation(); + + // If any of Lat, Long, or Elevation are provided, Lat and Long must both be provided. + if (StringUtils.isNotBlank(lat) || StringUtils.isNotBlank(lon) || StringUtils.isNotBlank(elevation)) { + if (StringUtils.isBlank(lat)) { + addRowError(ExperimentObservation.Columns.LAT, "Latitude must be provided for complete coordinate specification", validationErrors, rowNum); + } + if (StringUtils.isBlank(lon)) { + addRowError(ExperimentObservation.Columns.LONG, "Longitude must be provided for complete coordinate specification", validationErrors, rowNum); + } + } + + // Validate coordinate values + boolean latBadValue = false; + boolean lonBadValue = false; + boolean elevationBadValue = false; + double latDouble; + double lonDouble; + double elevationDouble; + + // Only check latitude format if not blank since already had previous error + if (StringUtils.isNotBlank(lat)) { + try { + latDouble = Double.parseDouble(lat); + if (latDouble < -90 || latDouble > 90) { + latBadValue = true; + } + } catch (NumberFormatException e) { + latBadValue = true; + } + } + + // Only check longitude format if not blank since already had previous error + if (StringUtils.isNotBlank(lon)) { + try { + lonDouble = Double.parseDouble(lon); + if (lonDouble < -180 || lonDouble > 180) { + lonBadValue = true; + } + } catch (NumberFormatException e) { + lonBadValue = true; + } + } + + if (StringUtils.isNotBlank(elevation)) { + try { + elevationDouble = Double.parseDouble(elevation); + } catch (NumberFormatException e) { + elevationBadValue = true; + } + } + + if (latBadValue) { + addRowError(ExperimentObservation.Columns.LAT, "Invalid Lat value (expected range -90 to 90)", validationErrors, rowNum); + } + + if (lonBadValue) { + addRowError(ExperimentObservation.Columns.LONG, "Invalid Long value (expected range -180 to 180)", validationErrors, rowNum); + } + + if (elevationBadValue) { + addRowError(ExperimentObservation.Columns.LONG, "Invalid Elevation value (numerals expected)", validationErrors, rowNum); + } + + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java new file mode 100644 index 000000000..7f0b741fe --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java @@ -0,0 +1,71 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.workflow; + +import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.AppendOverwritePhenotypesWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.CreateNewExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.List; + +@Singleton +public class ExperimentWorkflowFactory { + + private final Provider createNewExperimentWorkflowProvider; + private final Provider appendOverwritePhenotypesWorkflowProvider; + + @Inject + public ExperimentWorkflowFactory(Provider createNewExperimentWorkflowProvider, + Provider appendOverwritePhenotypesWorkflowProvider) { + this.createNewExperimentWorkflowProvider = createNewExperimentWorkflowProvider; + this.appendOverwritePhenotypesWorkflowProvider = appendOverwritePhenotypesWorkflowProvider; + } + + /** + * Retrieves the appropriate workflow based on the provided import context. Validation will be done + * in selected workflow, not here. For example will not check if file has ObsUnitIDs that all rows have one. + * We are just checking the basic condition for what type of workflow to return. + * + * @param context import context containing import rows + * @return the workflow to be used for processing the import rows + */ + public Workflow getWorkflow(ImportContext context) { + + List importRows = context.getImportRows(); + + boolean hasExpUnitObsUnitIDs = importRows.stream() + .anyMatch(row -> { + ExperimentObservation expRow = (ExperimentObservation) row; + return StringUtils.isNotBlank(expRow.getObsUnitID()); + }); + + if (hasExpUnitObsUnitIDs) { + long distinctCount = importRows.stream() + .map(row -> { + ExperimentObservation expRow = (ExperimentObservation) row; + return expRow.getObsUnitID(); + }) + .distinct() + .count(); + + if (distinctCount != importRows.size()) { + // If have ExpUnit ObsUnitIDs and there are duplicates -> Append / Update SubObsUnit Phenotypes + // TODO: different workflow for subobs units? + return appendOverwritePhenotypesWorkflowProvider.get(); + } else { + // If have ExpUnit ObsUnitIDs and all are unique -> Append / Update ExpUnit Phenotypes + return appendOverwritePhenotypesWorkflowProvider.get(); + } + + } else { + // No ObsUnitIDs so creating experiment or appending env + return createNewExperimentWorkflowProvider.get(); + // TODO: different workflow for appending envs? Would have a dependency on DAO to check for existing trial name + } + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/Workflow.java new file mode 100644 index 000000000..6c0384a0f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/Workflow.java @@ -0,0 +1,10 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.workflow; + +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; + +public interface Workflow { + + ProcessedData process(ImportContext context); + String getName(); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java new file mode 100644 index 000000000..17d6fe64e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java @@ -0,0 +1,83 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.workflow; + +import io.micronaut.context.ApplicationContext; +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.exceptions.NoSuchBeanException; +import io.micronaut.inject.qualifiers.Qualifiers; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.daos.ImportMappingWorkflowDAO; +import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; + +import javax.inject.Inject; +import java.util.Optional; +import java.util.UUID; + +@Factory +@Slf4j +public class WorkflowFactory { + + private final ImportMappingWorkflowDAO importMappingWorkflowDAO; + private final ApplicationContext applicationContext; + + @Inject + public WorkflowFactory(ImportMappingWorkflowDAO importMappingWorkflowDAO, + ApplicationContext applicationContext) { + this.importMappingWorkflowDAO = importMappingWorkflowDAO; + this.applicationContext = applicationContext; + } + + /** + * Produces the appropriate workflow instance based on the import context + * + * @param context the import context + * @return an Optional containing the workflow if id is not null, otherwise an empty Optional + * + * @throws IllegalStateException + * @throws NoSuchBeanException + */ + public Optional getWorkflow(UUID workflowId) { + + if (workflowId != null) { + // construct workflow from db record + Optional workflowOptional = importMappingWorkflowDAO.getWorkflowById(workflowId); + + ImportMappingWorkflow importMappingworkflow = workflowOptional.orElseThrow(() -> { + String msg = "Must have record in db for workflowId"; + log.error(msg); + return new IllegalStateException(msg); + }); + + // newer versions of micronaut have fancier ways to do this using annotations with provider but as + // far as I can tell it's not available in 2.5 + Workflow workflow; + try { + workflow = applicationContext.getBean(Workflow.class, Qualifiers.byName(importMappingworkflow.getBean())); + } catch (NoSuchBeanException e) { + log.error("Could not find workflow class implementation for bean: " + importMappingworkflow.getBean()); + throw e; + } + + return Optional.of(workflow); + } + + return Optional.empty(); + } +} diff --git a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql new file mode 100644 index 000000000..2c9d4d547 --- /dev/null +++ b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql @@ -0,0 +1,51 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + Table maps workflows to import mappings and provides required configuration options + + mapping_id - link to importer_mapping that this provides a workflow for + name - name that will be displayed on front end + bean - must match @Named("") annotation on Workflow class + position - for ordering records explicitly, wanted for front end default option and order + */ +CREATE TABLE importer_mapping_workflow +( + like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, + mapping_id UUID NOT NULL, + name TEXT NOT NULL, + bean TEXT NOT NULL, + position INTEGER NOT NULL +); + +ALTER TABLE importer_mapping_workflow + ADD FOREIGN KEY (mapping_id) REFERENCES importer_mapping (id); + +DO +$$ +DECLARE + exp_mapping_id UUID; +BEGIN + exp_mapping_id := (SELECT id FROM importer_mapping WHERE name = 'ExperimentsTemplateMap'); + +INSERT INTO public.importer_mapping_workflow (mapping_id, name, bean, position) +VALUES + (exp_mapping_id, 'Create new experiment', 'CreateNewExperimentWorkflow', 0), + (exp_mapping_id, 'Append experimental dataset', 'AppendOverwritePhenotypesWorkflow', 1), + (exp_mapping_id, 'Create new experimental environment', 'CreateNewEnvironmentWorkflow', 2); +END +$$; \ No newline at end of file