diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 59a43f17e..da9e70b3f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ name: maven build on: pull_request: - type: [opened, edited] + types: [opened, edited] jobs: build: diff --git a/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java b/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java index 3c7f1d706..7fdd58347 100644 --- a/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java +++ b/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java @@ -17,7 +17,9 @@ package org.breedinginsight.api.model.v1.response; -import lombok.*; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; import lombok.experimental.Accessors; import java.util.ArrayList; diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java index 6eb4c2761..fea32b569 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java @@ -183,6 +183,27 @@ public List getObservationsByStudyName(List studyNames ); } + /** + * Retrieves a list of observations based on their database IDs and a specific program. + * + * @param dbIds A list of database IDs representing the observations to retrieve. + * @param program The Program object for which the observations belong. + * @return A List of BrAPIObservation objects filtered by the provided database IDs. + * @throws ApiException if an error occurs during the retrieval process. + */ + public List getObservationsByDbIds(List dbIds, Program program) throws ApiException { + // Check if the dbIds list is empty and return an empty list if so + if(dbIds.isEmpty()) { + return Collections.emptyList(); + } + + // Filter the observations based on the provided program ID and the provided list of dbIds + // Collect the filtered observations into a List and return the result + return getProgramObservations(program.getId()).values().stream() + .filter(o -> dbIds.contains(o.getObservationDbId())) + .collect(Collectors.toList()); + } + public List getObservationsByTrialDbId(List trialDbIds, Program program) throws ApiException { if(trialDbIds.isEmpty()) { return Collections.emptyList(); diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index caab88879..6b6fcbcae 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -390,8 +390,8 @@ private void addObsVarDataToRow( Trait var, Program program) { String varName = Utilities.removeProgramKey(obs.getObservationVariableName(), program.getKey()); - if (!(obs.getValue().equalsIgnoreCase("NA")) && (var.getScale().getDataType().equals(DataType.NUMERICAL) || - var.getScale().getDataType().equals(DataType.DURATION))) { + if (!("NA".equalsIgnoreCase(obs.getValue())) && (DataType.NUMERICAL.equals(var.getScale().getDataType()) || + DataType.DURATION.equals(var.getScale().getDataType()))) { row.put(varName, Double.parseDouble(obs.getValue())); } else { row.put(varName, obs.getValue()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index e0a61117a..c9ea3ec39 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -37,6 +37,7 @@ import org.breedinginsight.api.model.v1.response.metadata.StatusCode; import org.breedinginsight.api.v1.controller.metadata.AddMetadata; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.ImportConfigManager; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; import org.breedinginsight.brapps.importer.services.FileImportService; @@ -76,7 +77,7 @@ public HttpResponse>> getImportTypes Pagination pagination = new Pagination(configs.size(), 1, 1, 0); Metadata metadata = new Metadata(pagination, metadataStatus); - Response> response = new Response(metadata, new DataResponse<>(configs)); + Response> response = new Response<>(metadata, new DataResponse<>(configs)); return HttpResponse.ok(response); } @@ -95,7 +96,7 @@ public HttpResponse>> getMappings(@PathVari Pagination pagination = new Pagination(result.size(), 1, 1, 0); Metadata metadata = new Metadata(pagination, metadataStatus); - Response> response = new Response(metadata, new DataResponse<>(result)); + Response> response = new Response<>(metadata, new DataResponse<>(result)); return HttpResponse.ok(response); } catch (DoesNotExistException e) { log.info(e.getMessage()); @@ -116,7 +117,7 @@ public HttpResponse> createMapping(@PathVariable UUID pr try { AuthenticatedUser actingUser = securityService.getUser(); ImportMapping result = fileImportService.createMapping(programId, actingUser, file); - Response response = new Response(result); + Response response = new Response<>(result); return HttpResponse.ok(response); } catch (DoesNotExistException e) { log.info(e.getMessage()); @@ -140,7 +141,7 @@ public HttpResponse> editMappingFile(@PathVariable UUID try { AuthenticatedUser actingUser = securityService.getUser(); ImportMapping result = fileImportService.updateMappingFile(programId, mappingId, actingUser, file); - Response response = new Response(result); + Response response = new Response<>(result); return HttpResponse.ok(response); } catch (DoesNotExistException e) { log.info(e.getMessage()); @@ -165,7 +166,7 @@ public HttpResponse> editMapping(@PathVariable UUID prog try { AuthenticatedUser actingUser = securityService.getUser(); ImportMapping result = fileImportService.updateMapping(programId, actingUser, mappingId, mapping, validate); - Response response = new Response(result); + Response response = new Response<>(result); return HttpResponse.ok(response); } catch (DoesNotExistException e) { log.error(e.getMessage(), e); @@ -205,7 +206,30 @@ public HttpResponse>> getSystemMappings(@Nu Pagination pagination = new Pagination(result.size(), result.size(), 1, 0); Metadata metadata = new Metadata(pagination, metadataStatus); - Response> response = new Response(metadata, new DataResponse<>(result)); + Response> response = new Response<>(metadata, new DataResponse<>(result)); + return HttpResponse.ok(response); + } + + @Get("/import/mappings/{mappingId}/workflows") + @Produces(MediaType.APPLICATION_JSON) + @AddMetadata + @Secured(SecurityRule.IS_ANONYMOUS) + public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { + + List workflows = null; + try { + workflows = fileImportService.getWorkflowsForSystemMapping(mappingId); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } + + List metadataStatus = new ArrayList<>(); + metadataStatus.add(new Status(StatusCode.INFO, "Successful Query")); + Pagination pagination = new Pagination(workflows.size(), workflows.size(), 1, 0); + Metadata metadata = new Metadata(pagination, metadataStatus); + + Response> response = new Response<>(metadata, new DataResponse<>(workflows)); return HttpResponse.ok(response); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java index f9d55bd20..74928dc10 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java @@ -62,7 +62,7 @@ public HttpResponse> uploadData(@PathVariable UUID prog try { AuthenticatedUser actingUser = securityService.getUser(); ImportResponse result = fileImportService.uploadData(programId, mappingId, actingUser, file); - Response response = new Response(result); + Response response = new Response<>(result); return HttpResponse.ok(response); } catch (DoesNotExistException e) { log.error(e.getMessage(), e); @@ -94,7 +94,7 @@ public HttpResponse> getUploadData(@PathVariable UUID p try { AuthenticatedUser actingUser = securityService.getUser(); Pair result = fileImportService.getDataUpload(uploadId, mapping); - Response response = new Response(result.getRight()); + Response response = new Response<>(result.getRight()); if (result.getLeft().equals(HttpStatus.ACCEPTED)) { return HttpResponse.ok(response).status(result.getLeft()); } else { @@ -114,7 +114,7 @@ public HttpResponse> commitData(@PathVariable UUID prog @PathVariable UUID uploadId, @Body @Nullable Map userInput) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, actingUser, userInput, true); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, null, actingUser, userInput, true); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { @@ -140,8 +140,61 @@ public HttpResponse> previewData(@PathVariable UUID pro @PathVariable UUID uploadId) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, actingUser, null, false); - Response response = new Response(result); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, null, actingUser, null, false); + Response response = new Response<>(result); + return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + return HttpResponse.notFound(); + } catch (AuthorizationException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.FORBIDDEN, e.getMessage()); + } catch (UnprocessableEntityException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } catch (HttpStatusException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(e.getStatus(), e.getMessage()); + } + } + + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflow}/data/{uploadId}/preview") + @Produces(MediaType.APPLICATION_JSON) + @AddMetadata + @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) + public HttpResponse> previewData(@PathVariable UUID programId, @PathVariable UUID mappingId, + @PathVariable String workflow, @PathVariable UUID uploadId) { + try { + AuthenticatedUser actingUser = securityService.getUser(); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflow, actingUser, null, false); + Response response = new Response<>(result); + return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + return HttpResponse.notFound(); + } catch (AuthorizationException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.FORBIDDEN, e.getMessage()); + } catch (UnprocessableEntityException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } catch (HttpStatusException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(e.getStatus(), e.getMessage()); + } + } + + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflow}/data/{uploadId}/commit") + @Produces(MediaType.APPLICATION_JSON) + @AddMetadata + @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) + public HttpResponse> commitData(@PathVariable UUID programId, @PathVariable UUID mappingId, + @PathVariable String workflow, @PathVariable UUID uploadId, + @Body @Nullable Map userInput) { + try { + AuthenticatedUser actingUser = securityService.getUser(); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflow, actingUser, userInput, true); + Response response = new Response<>(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { log.error(e.getMessage(), e); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/Workflow.java new file mode 100644 index 000000000..807f8387f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/Workflow.java @@ -0,0 +1,10 @@ +package org.breedinginsight.brapps.importer.model; + +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; + +public interface Workflow { + + ProcessedData process(ImportContext context); + String getName(); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index 1d520371c..3f25d99e6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -17,22 +17,15 @@ package org.breedinginsight.brapps.importer.model.imports; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.ValidatorException; -import tech.tablesaw.api.Table; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import java.util.List; public interface BrAPIImportService { String getImportTypeId(); BrAPIImport getImportClass(); + List getWorkflows(); default String getInvalidIntegerMsg(String columnName) { return String.format("Column name \"%s\" must be integer type, but non-integer type provided.", columnName); } @@ -48,6 +41,6 @@ default String getMissingUserInputMsg(String fieldName) { default String getWrongUserInputDataTypeMsg(String fieldName, String typeName) { return String.format("User input, \"%s\" must be an %s", fieldName, typeName); } - ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + ImportPreviewResponse process(ImportServiceContext context) throws Exception; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java new file mode 100644 index 000000000..a8dce4090 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -0,0 +1,73 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.imports; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; +import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; + +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.List; +import java.util.Optional; + +@Singleton +@Slf4j +public abstract class DomainImportService implements BrAPIImportService { + private final Workflow workflowNavigator; + + + public DomainImportService(Workflow workflowNavigator) + { + this.workflowNavigator = workflowNavigator; + } + + @Override + public String getMissingColumnMsg(String columnName) { + return "Column heading does not match template or ontology"; + } + @Override + public List getWorkflows() { + return workflowNavigator.getWorkflows(); + } + + @Override + public ImportPreviewResponse process(ImportServiceContext context) + throws Exception { + + Optional.ofNullable(context.getWorkflow()) + .filter(workflow -> !workflow.isEmpty()) + .ifPresent(workflow -> log.info("Workflow: " + workflow)); + + + Optional result = workflowNavigator.process(context); + + // Throw any exceptions caught during workflow processing + if (result.flatMap(ImportWorkflowResult::getCaughtException).isPresent()) { + throw result.flatMap(ImportWorkflowResult::getCaughtException).get(); + } + + return result.flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); + } +} + diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java new file mode 100644 index 000000000..4d052cd33 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -0,0 +1,42 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.imports; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import tech.tablesaw.api.Table; + +import java.util.List; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ImportServiceContext { + private String workflow; + private List brAPIImports; + private Table data; + private Program program; + private ImportUpload upload; + private User user; + private boolean commit; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index cd795564a..9e27d0e20 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -18,16 +18,15 @@ package org.breedinginsight.brapps.importer.model.imports.experimentObservation; import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.DomainImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; -import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import tech.tablesaw.api.Table; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import javax.inject.Inject; import javax.inject.Provider; @@ -36,18 +35,14 @@ @Singleton @Slf4j -public class ExperimentImportService implements BrAPIImportService { +public class ExperimentImportService extends DomainImportService { private final String IMPORT_TYPE_ID = "ExperimentImport"; - private final Provider experimentProcessorProvider; - private final Provider processorManagerProvider; - @Inject - public ExperimentImportService(Provider experimentProcessorProvider, Provider processorManagerProvider) + public ExperimentImportService(ExperimentWorkflowNavigator workflowNavigator) { - this.experimentProcessorProvider = experimentProcessorProvider; - this.processorManagerProvider = processorManagerProvider; + super(workflowNavigator); } @Override @@ -60,20 +55,6 @@ public String getImportTypeId() { return IMPORT_TYPE_ID; } - @Override - public String getMissingColumnMsg(String columnName) { - return "Column heading does not match template or ontology"; - } - @Override - public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) - throws Exception { - - ImportPreviewResponse response = null; - List processors = List.of(experimentProcessorProvider.get()); - response = processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); - return response; - - } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index b4eac6b96..0caebe65e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -21,7 +21,9 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.GermplasmProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; @@ -32,6 +34,7 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import java.util.ArrayList; import java.util.List; @Singleton @@ -56,18 +59,29 @@ public GermplasmImport getImportClass() { return new GermplasmImport(); } + @Override + public List getWorkflows() { + return new ArrayList<>(); + } + @Override public String getImportTypeId() { return IMPORT_TYPE_ID; } @Override - public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + public ImportPreviewResponse process(ImportServiceContext context) throws Exception { ImportPreviewResponse response = null; List processors = List.of(germplasmProcessorProvider.get()); - response = processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); + response = processorManagerProvider.get().process(context.getBrAPIImports(), + processors, + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); return response; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java index 434626e68..eb7328ecf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java @@ -21,7 +21,9 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.brapps.importer.services.processors.SampleSubmissionProcessor; @@ -32,6 +34,7 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import java.util.ArrayList; import java.util.List; @Singleton @@ -59,13 +62,19 @@ public BrAPIImport getImportClass() { } @Override - public ImportPreviewResponse process(List brAPIImports, - Table data, - Program program, - ImportUpload upload, - User user, - Boolean commit) throws Exception { + public List getWorkflows() { + return new ArrayList<>(); + } + + @Override + public ImportPreviewResponse process(ImportServiceContext context) throws Exception { List processors = List.of(sampleProcessorProvider.get()); - return processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); + return processorManagerProvider.get().process(context.getBrAPIImports(), + processors, + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/response/ImportPreviewResponse.java b/src/main/java/org/breedinginsight/brapps/importer/model/response/ImportPreviewResponse.java index b46fc6335..23fec70c6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/response/ImportPreviewResponse.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/response/ImportPreviewResponse.java @@ -17,6 +17,7 @@ package org.breedinginsight.brapps.importer.model.response; +import lombok.Builder; import lombok.Getter; import lombok.Setter; import org.breedinginsight.brapps.importer.model.imports.PendingImport; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java new file mode 100644 index 000000000..d0f094cf2 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java @@ -0,0 +1,23 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +@FunctionalInterface +public interface ExperimentWorkflow extends Workflow { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java new file mode 100644 index 000000000..9be01f51b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java @@ -0,0 +1,23 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +@FunctionalInterface +public interface GermplasmWorkflow extends Workflow { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java new file mode 100644 index 000000000..121ef6b19 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java @@ -0,0 +1,47 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import tech.tablesaw.api.Table; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ImportContext { + private UUID workflowId; + private ImportUpload upload; + private List importRows; + private Map mappedBrAPIImport; + private Table data; + private Program program; + private User user; + private boolean commit; +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java new file mode 100644 index 000000000..30dbac06a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java @@ -0,0 +1,31 @@ +///* +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +public class ImportWorkflow { + private String id; + private String name; + private int order; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java new file mode 100644 index 000000000..f59b83b55 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java @@ -0,0 +1,34 @@ +///* +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; + +import java.util.Optional; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +public class ImportWorkflowResult { + private ImportWorkflow workflow; + private Optional importPreviewResponse; + private Optional caughtException; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java new file mode 100644 index 000000000..f9f8196c2 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -0,0 +1,30 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; + +import java.util.Map; + +@Data +@ToString +@NoArgsConstructor +public class ProcessedData { + private Map statistics; +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java new file mode 100644 index 000000000..a0a81f123 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java @@ -0,0 +1,23 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +@FunctionalInterface +public interface SampleSubmissionWorkflow extends Workflow { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java new file mode 100644 index 000000000..b82bbd690 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -0,0 +1,52 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import io.micronaut.core.order.Ordered; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * This Functional Interface represents a Workflow that can be executed as part of an import process. + * It extends the Ordered interface to allow workflows to be ordered in a sequence. + */ +@FunctionalInterface +public interface Workflow extends Ordered { + + /** + * Process method that defines the logic to be executed as part of the workflow. + * + * @param context the ImportServiceContext object containing necessary information for the workflow + * @return an Optional of ImportWorkflowResult representing the result of the workflow execution + */ + Optional process(ImportServiceContext context); + + /** + * Default method to get a list of workflows. + * This method provides a default implementation returning an empty list. + * + * @return a List of ImportWorkflow containing workflows + */ + default List getWorkflows() { + // Default implementation for getWorkflows method + return new ArrayList<>(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 05c48601d..fc660d6da 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -36,10 +36,12 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingProgramEntity; import org.breedinginsight.model.Program; @@ -322,7 +324,7 @@ public ImportResponse uploadData(UUID programId, UUID mappingId, AuthenticatedUs return response; } - public ImportResponse updateUpload(UUID programId, UUID uploadId, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws + public ImportResponse updateUpload(UUID programId, UUID uploadId, String workflow, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws DoesNotExistException, UnprocessableEntityException, AuthorizationException { Program program = validateRequest(programId, actingUser); @@ -372,7 +374,7 @@ public ImportResponse updateUpload(UUID programId, UUID uploadId, AuthenticatedU } else { brAPIImportList = mappingManager.map(mappingConfig, data); } - processFile(brAPIImportList, data, program, upload, user, commit, importService, actingUser); + processFile(workflow, brAPIImportList, data, program, upload, user, commit, importService, actingUser); } catch (UnprocessableEntityException e) { log.error(e.getMessage(), e); ImportProgress progress = upload.getProgress(); @@ -418,13 +420,22 @@ public ImportUpload setDynamicColumns(ImportUpload newUpload, Table data, Import return newUpload; } - private void processFile(List finalBrAPIImportList, Table data, Program program, - ImportUpload upload, User user, Boolean commit, BrAPIImportService importService, - AuthenticatedUser actingUser) { + private void processFile(String workflow, List finalBrAPIImportList, Table data, Program program, + ImportUpload upload, User user, Boolean commit, BrAPIImportService importService, + AuthenticatedUser actingUser) { // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { - importService.process(finalBrAPIImportList, data, program, upload, user, commit); + ImportServiceContext context = ImportServiceContext.builder() + .workflow(workflow) + .brAPIImports(finalBrAPIImportList) + .data(data) + .program(program) + .upload(upload) + .user(user) + .commit(commit) + .build(); + importService.process(context); } catch (UnprocessableEntityException e) { log.error(e.getMessage(), e); ImportProgress progress = upload.getProgress(); @@ -559,4 +570,24 @@ public List getSystemMappingByName(String name) { List importMappings = importMappingDAO.getSystemMappingByName(name); return importMappings; } + + /** + * Retrieves the list of import workflows associated with a specific system mapping. + * + * @param mappingId The ID of the system mapping for which to retrieve workflows + * @return A list of ImportWorkflow objects representing the workflows for the specified system mapping + * @throws DoesNotExistException If the system mapping with the given ID does not exist + */ + public List getWorkflowsForSystemMapping(UUID mappingId) throws DoesNotExistException { + // Retrieve the import mapping configuration based on the provided mapping ID + ImportMapping mappingConfig = importMappingDAO.getMapping(mappingId) + .orElseThrow(() -> new DoesNotExistException("Cannot find mapping config associated with upload.")); + + // Get the import service associated with the import type ID from the configuration manager + BrAPIImportService importService = configManager.getImportServiceById(mappingConfig.getImportTypeId()) + .orElseThrow(() -> new DoesNotExistException("Config with that id does not exist")); + + // Retrieve and return the list of import workflows for the specified system mapping + return importService.getWorkflows(); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index f27d89338..c761c358c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -433,6 +433,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program if (observation == null) { throw new Exception("Null observation"); } + BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); if (updatedObs == null) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java new file mode 100644 index 000000000..6ef89f326 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -0,0 +1,470 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonSyntaxException; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; +import io.reactivex.functions.Function; +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + + + +@Singleton +public class ExperimentUtilities { + + public static final CharSequence COMMA_DELIMITER = ","; + public static final String TIMESTAMP_PREFIX = "TS:"; + + Gson gson; + + public ExperimentUtilities() { + this.gson = new Gson(); + } + + /** + * Checks if the provided list contains any invalid members for the specified class. + * + * @param list The list to be checked for invalid members + * @param clazz The class to check for instance validity + * @return true if the list is null, empty, or contains any member that is not an instance of the specified class; false otherwise + */ + public boolean isInvalidMemberListForClass(List list, Class clazz) { + // Check if the input list is null, empty, or contains any member that is not an instance of the specified class + return list == null || list.isEmpty() || !list.stream().allMatch(clazz::isInstance); + } + + /** + * This method creates a deep copy of an object using Gson library in Java 8. + * It takes an object of type T and its corresponding class to clone. + * @param obj the object to clone + * @param clazz the class of the object to clone + * @return an Optional containing the cloned object if successful, otherwise an empty Optional + * @throws JsonSyntaxException if there is an issue with JSON syntax during the cloning process + */ + public Optional clone(T obj, Class clazz) { + try { + // Convert the object to JSON string and then parse it back to the specified class + return Optional.ofNullable(gson.fromJson(gson.toJson(obj), clazz)); + } catch (JsonSyntaxException e) { + // Return an empty Optional if there is a JsonSyntaxException + return Optional.empty(); + } + } + + /** + * Retrieves a list of new objects of type T from the provided map of pending import objects by filtering out null previews and objects with state other than NEW, + * mapping the BrAPI object from each preview, cloning it to the specified class type, and collecting the non-empty results into a list. + * + * @param objectsByName a map of pending import objects with V keys and PendingImportObject values + * @param clazz the target class type for cloning the BrAPI object + * @param the type of new objects to be retrieved + * @param the type of keys in the map of pending import objects + * @return a list of cloned new objects of type T extracted from the input map + */ + public List getNewObjects(Map> objectsByName, Class clazz) { + return objectsByName.values().stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW) + .map(PendingImportObject::getBrAPIObject) + .map(b->clone(b, clazz)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + + /** + * Copies mutated objects from a cache map to a new list. + * Only objects with ImportObjectState MUTATED are included in the copied list. + * + * @param pendingCacheMap a map containing PendingImportObject objects to be copied + * @param clazz a Class object representing the type of objects to be copied + * @return a List of copied objects of type T + */ + public List copyMutationsFromCache(Map> pendingCacheMap, Class clazz) { + return pendingCacheMap.values().stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED) + .map(PendingImportObject::getBrAPIObject) + .map(b -> clone(b, clazz)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + + /** + * Copies the pending BrAPI objects from the workflow cache map based on the provided import object status. + * + * @param pendingCacheMap a map containing pending import objects with generic values V and T + * @param clazz the class type of the BrAPI object to be cloned + * @param status the import object state to filter the pending objects + * @return a list of cloned BrAPI objects that are in the specified import object state + */ + public List copyWorkflowCachePendingBrAPIObjects(Map> pendingCacheMap, + Class clazz, + ImportObjectState status) { + // Filter the pending import objects by checking if the object is not null and has the specified status + return pendingCacheMap.values().stream() + .filter(preview -> preview != null && preview.getState() == status) + // Map each pending import object to its corresponding BrAPI object + .map(PendingImportObject::getBrAPIObject) + // Clone the BrAPI object with the provided class type + .map(brApiObject -> clone(brApiObject, clazz)) + // Filter out any empty optionals + .filter(Optional::isPresent) + // Unwrap the optional value + .map(Optional::get) + // Collect the cloned BrAPI objects into a list + .collect(Collectors.toList()); + } + + /** + * Retrieves mutations by object ID from a Map of PendingImportObject, filtering based on the object state and applying a DB ID filter. + * + * @param objectsByName A Map with values of type PendingImportObject, used for retrieving the mutations. + * @param dbIdFilter A Function that filters the objects based on their DB ID. + * @param clazz The Class type for the objects in the Map. + * @param Type parameter for the objects in the Map. + * @param Type parameter for the keys in the Map. + * @return A Map of String keys (DB IDs) and objects of type T as values based on the filter logic. + * @throws RuntimeException if an exception occurs while applying the DB ID filter to an object. + */ + public Map getMutationsByObjectId(Map> objectsByName, Function dbIdFilter, Class clazz) { + return objectsByName.values().stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED) + .map(PendingImportObject::getBrAPIObject) + .map(b -> clone(b, clazz)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors + .toMap(brapiObj -> { + try { + return dbIdFilter.apply(brapiObj); + } catch (Exception e) { + throw new RuntimeException(e); + } + }, + brapiObj -> brapiObj)); + } + + /** + * Converts a list of BrAPI imports to a list of ExperimentObservations. + * + * This function takes a list of BrAPIImport objects representing trial imports and converts them + * to ExperimentObservation objects. It utilizes Java 8 stream API to map each BrAPIImport object + * to an ExperimentObservation object and collects the results into a new list. + * + * @param importRows a list of BrAPIImport objects representing trial imports to be converted + * @return a list of ExperimentObservation objects containing the converted data + */ + public static List importRowsToExperimentObservations(List importRows) { + return importRows.stream() + .map(trialImport -> (ExperimentObservation) trialImport) + .collect(Collectors.toList()); + } + + /** + * This method generates a unique key for an observation unit based on the environment and experimental unit ID. + * + * @param importRow the ExperimentObservation object containing the environment and experimental unit ID + * @return a String representing the unique key for the observation unit + */ + public static String createObservationUnitKey(ExperimentObservation importRow) { + // Extract the environment and experimental unit ID from the ExperimentObservation object + // and pass them to the createObservationUnitKey method + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); + } + + /** + * Create Observation Unit Key + * + * This method takes in the name of a study and the name of an observation unit and concatenates them to create a unique key. + * + * @param studyName The name of the study + * @param obsUnitName The name of the observation unit + * @return A string representing the unique key formed by concatenating the study name and observation unit name + */ + public static String createObservationUnitKey(String studyName, String obsUnitName) { + // Concatenate the study name and observation unit name to create the unique key + return studyName + obsUnitName; + } + + /** + * Returns the single value from the provided map if it contains exactly one entry. + * + * @param map the map from which to extract the single value + * @param message the message to be included in the exception thrown if the map does not contain exactly one entry + * @return the single value from the map + * @throws UnprocessableEntityException if the map does not contain only one entry + */ + public V getSingleEntryValue(Map map, String message) throws UnprocessableEntityException { + if (map.size() != 1) { + throw new UnprocessableEntityException(message); + } + return map.values().iterator().next(); + } + +// Module/Script-level documentation + /** + * This method is used to retrieve the value from a map if the map contains only one entry. + * This method is particularly useful when dealing with scenarios where a single entry is expected. + * It throws an UnprocessableEntityException if the map does not contain exactly one entry, providing a custom error message for clarity. + * Usage: + * Map exampleMap = new HashMap<>(); + * exampleMap.put("exampleKey", 5); + * Integer value = getSingleEntryValue(exampleMap, "Map should contain exactly one entry."); + */ + + /** + * Retrieves the single value from a given map, if the map contains exactly one key-value pair. + * + * @param map The map from which to retrieve the single value. + * @return An Optional containing the single value if the map contains only one key-value pair, + * or an empty Optional if the map is empty or contains more than one entry. + * + * Input: + * - map: The map from which to retrieve the single value. + * + * Output: + * - An Optional containing the single value if the map contains exactly one key-value pair, + * or an empty Optional if the map is empty or contains more than one entry. + * + * Side Effects: + * - None + * + * Usage: + * if you have a map and you want to retrieve the single value associated with a key, you can use + * this function to ensure that the map contains only one entry before retrieving the value. + */ + public static Optional getSingleEntryValue(Map map) { + Optional value = Optional.empty(); + if (map.size() == 1) { + value = Optional.ofNullable(map.values().iterator().next()); + } + return value; + } + + /* + * Add a given year to the additionalInfo field of the BrAPIStudy, if it does not already exist. + * + * @param program the program to which the study belongs + * @param study the BrAPIStudy object to which the year should be added + * @param year the year to be added to the additionalInfo field + * + * This method checks if the additionalInfo field of the BrAPIStudy object is null, and if so, initializes it with a new JsonObject. + * Then, it checks if the ENV_YEAR key already exists in the additionalInfo object, and if not, adds the given year with the key ENV_YEAR. + * + * @return void + */ + + /* Module Description + * This module contains a method that adds a given year to the additionalInfo field of a BrAPIStudy object within a program's context. + * The purpose of this method is to provide a convenient way to store and retrieve additional information related to the study. + * + * Usage: + * To add a year to the additionalInfo field of a BrAPIStudy object, call this method passing the program, study, and year as parameters. + */ + + /* Side effects: + * This method mutates the state of the BrAPIStudy object by adding the given year to its additionalInfo field. + */ + + /* + * this will add the given year to the additionalInfo field of the BrAPIStudy (if it does not already exist) + * */ + public void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study, String year) { + JsonObject additionalInfo = study.getAdditionalInfo(); + if (additionalInfo==null){ + additionalInfo = new JsonObject(); + study.setAdditionalInfo(additionalInfo); + } + if( additionalInfo.get(BrAPIAdditionalInfoFields.ENV_YEAR)==null) { + additionalInfo.addProperty(BrAPIAdditionalInfoFields.ENV_YEAR, year); + } + } + + /** + * This method is responsible for collating unique ObsUnit IDs from the provided context data. + * + * @param context the AppendOverwriteMiddlewareContext containing the import rows to process + * @return a Set of unique ObsUnit IDs collated from the import rows + * @throws IllegalStateException if any ObsUnit ID is repeated in the import rows + * @throws HttpStatusException if there is a mix of ObsUnit IDs for some but not all rows + */ + public static Set collateReferenceOUIds(AppendOverwriteMiddlewareContext context) { + // Initialize variables to track the presence of ObsUnit IDs + Set referenceOUIds = new HashSet<>(); + boolean hasNoReferenceUnitIds = true; + boolean hasAllReferenceUnitIds = true; + + // Iterate through the import rows to process ObsUnit IDs + for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + + // Check if ObsUnitID is blank + if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { + // Set flag to indicate missing ObsUnit ID for current row + hasAllReferenceUnitIds = false; + } else if (referenceOUIds.contains(importRow.getObsUnitID())) { + // Throw exception if ObsUnitID is repeated + throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); + } else { + // Add ObsUnitID to referenceOUIds + referenceOUIds.add(importRow.getObsUnitID()); + // Set flag to indicate presence of ObsUnit ID + hasNoReferenceUnitIds = false; + } + } + + if (!hasNoReferenceUnitIds && !hasAllReferenceUnitIds) { + // Throw exception if there is a mix of ObsUnit IDs for some but not all rows + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, ExpImportProcessConstants.ErrMessage.MISSING_OBS_UNIT_ID_ERROR); + } + + return referenceOUIds; + } + + /** + * This method sorts a list of items based on a list of sorted fields in ascending order using Java 8 functionality. + * + * @param sortedFields a list of strings representing the fields to sort by + * @param unsortedItems a list of items of generic type T to be sorted + * @param fieldGetter a Function object that extracts the string field from an item of type T + * @return a sorted list of items of type T based on the specified fields + * @throws RuntimeException if there are any exceptions encountered during sorting + */ + public List sortByField(List sortedFields, List unsortedItems, Function fieldGetter) { + // Create a case-insensitive map to store the sort order of fields + CaseInsensitiveMap sortOrder = new CaseInsensitiveMap<>(); + + // Populate the sortOrder map with the fields and their respective indices from the sortedFields list + for (int i = 0; i < sortedFields.size(); i++) { + sortOrder.put(sortedFields.get(i), i); + } + + // Sort the unsortedItems list using a lambda expression to compare items based on the order of specified fields + unsortedItems.sort((i1, i2) -> { + try { + // Extract the field values of the items using the fieldGetter function + String field1 = fieldGetter.apply(i1); + String field2 = fieldGetter.apply(i2); + + // Compare the indices of the fields in sortOrder and return the result + return Integer.compare(sortOrder.get(field1), sortOrder.get(field2)); + } catch (Exception e) { + // Throw a runtime exception if any error occurs during sorting + throw new RuntimeException(e); + } + }); + + // Return the sorted list of items + return unsortedItems; + } + + /** + * Constructs a list of BrAPIExternalReference objects for various entities using the given parameters. + * + * @param program the program entity for which external references are to be constructed + * @param referenceSourceBaseName the base name for the reference source + * @param trialId the UUID of the trial entity + * @param datasetId the UUID of the dataset entity + * @param studyId the UUID of the study entity + * @param obsUnitId the UUID of the observation unit entity + * @param observationId the UUID of the observation entity + * @return a list of BrAPIExternalReference objects representing the external references + */ + public List constructBrAPIExternalReferences( + Program program, String referenceSourceBaseName, UUID trialId, UUID datasetId, UUID studyId, UUID obsUnitId, UUID observationId) { + List refs = new ArrayList<>(); + + // Add reference for the program entity + addReference(refs, program.getId(), referenceSourceBaseName, ExternalReferenceSource.PROGRAMS); + + // Add reference for the trial entity if available + if (trialId != null) { + addReference(refs, trialId, referenceSourceBaseName, ExternalReferenceSource.TRIALS); + } + + // Add reference for the dataset entity if available + if (datasetId != null) { + addReference(refs, datasetId, referenceSourceBaseName, ExternalReferenceSource.DATASET); + } + + // Add reference for the study entity if available + if (studyId != null) { + addReference(refs, studyId, referenceSourceBaseName, ExternalReferenceSource.STUDIES); + } + + // Add reference for the observation unit entity if available + if (obsUnitId != null) { + addReference(refs, obsUnitId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATION_UNITS); + } + + // Add reference for the observation entity if available + if (observationId != null) { + addReference(refs, observationId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATIONS); + } + + return refs; + } + + /** + * Adds a new reference to the given list of BrAPIExternalReference objects. + * + * @param refs the list of BrAPIExternalReference objects to which the new reference will be added + * @param uuid the UUID to set as the reference ID for the new BrAPIExternalReference + * @param referenceBaseNameSource the base name for the reference source + * @param refSourceName the source of the external reference + */ + private void addReference(List refs, UUID uuid, String referenceBaseNameSource, ExternalReferenceSource refSourceName) { + // Create a new BrAPIExternalReference object + BrAPIExternalReference reference = new BrAPIExternalReference(); + + // Set the reference source as a combination of reference base name source and external reference source + reference.setReferenceSource(String.format("%s/%s", referenceBaseNameSource, refSourceName.getName())); + + // Set the reference ID as the UUID converted to a string + reference.setReferenceID(uuid.toString()); + + // Add the new reference to the list of references + refs.add(reference); + } + /** + * Module overview: This module provides a method to add a new reference to a list of BrAPIExternalReference objects. + * The method takes in the list of references, a UUID, reference base name source, and external reference source. + * It creates a new BrAPIExternalReference object, sets the reference source and ID, and adds it to the list of references. + * This function is useful when dealing with external references in BrAPI-related operations. + * Usage: Call this method with the required parameters to add a new reference to the existing list of references. + */ +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java new file mode 100644 index 000000000..520fc53d5 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -0,0 +1,144 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment; + +import io.micronaut.context.annotation.Primary; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; + +import javax.inject.Singleton; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +@Primary +@Singleton +public class ExperimentWorkflowNavigator implements ExperimentWorkflow { + private final List workflows; + + /** Micronaut scans and collects in a List all instances of ExperimentWorkflow not annotated as @Primary and + * automatically makes the list available to inject into the constructor, which is set here to the workflows field. + * The order in the list is determined by the sort value returned from each instance by calling getOrder(). + * Instances returning a lower sort value will appear in the list before instances returning higher sort values. + */ + public ExperimentWorkflowNavigator(List workflows) { + this.workflows = workflows; + } + + /** + * Process the import service context by executing a series of workflows in order + * + * This method iterates over the list of workflows provided, executing each workflow's process method + * with the given import service context. It then filters out empty results and returns the first non-empty result. + * + * @param context The import service context containing the data to be processed + * @return An Optional containing the first non-empty ImportWorkflowResult from the executed workflows, or an empty Optional if no non-empty result is found + */ + @Override + public Optional process(ImportServiceContext context) { + /** + * Have each workflow in order process the context, returning the first non-empty result + */ + return workflows.stream() + .map(workflow->workflow.process(context)) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst(); + } + + /** + * Retrieves a list of ImportWorkflow objects containing metadata about each workflow that processed the import context. + * + * @return List of ImportWorkflow objects with workflow metadata + */ + public List getWorkflows() { + List workflowSummaryList = workflows.stream() + .map(workflow -> workflow.process(null)) // Process each workflow with a null context + .filter(Optional::isPresent) // Filter out any workflows that do not return a result + .map(Optional::get) // Extract the result from Optional + .map(result -> result.getWorkflow()) // Retrieve the workflow metadata + .collect(Collectors.toList()); // Collect the workflow metadata into a list + + // Set the order field for each workflow based on its position in the list + for (int i = 0; i < workflowSummaryList.size(); i++) { + workflowSummaryList.get(i).setOrder(i); // Set the order for each workflow + } + + return workflowSummaryList; // Return the list of workflow metadata + } + + /** + * The Workflow enum represents different workflow types that can be associated with an experiment. + */ + public enum Workflow { + + /** + * Represents a new observation workflow where a new experiment is created. + * ID: "new-experiment" + * Name: "Create new experiment" + */ + NEW_OBSERVATION("new-experiment","Create new experiment"), + + /** + * Represents an append or overwrite workflow where experimental dataset is appended. + * ID: "append-dataset" + * Name: "Append experimental dataset" + */ + APPEND_OVERWRITE("append-dataset", "Append experimental dataset"); + + private String id; + private String name; + + /** + * Constructor for the Workflow enum to initialize ID and Name. + * @param id The ID of the workflow. + * @param name The name of the workflow. + */ + Workflow(String id, String name) { + this.id = id; + this.name = name; + } + + /** + * Get the ID of the workflow. + * @return The ID of the workflow. + */ + public String getId() { + return id; + } + + /** + * Get the name of the workflow. + * @return The name of the workflow. + */ + public String getName() { + return name; + } + + /** + * Check if the given value is equal to the ID of the workflow. + * @param value The value to compare with the workflow ID. + * @return true if the value is equal to the ID, false otherwise. + */ + public boolean isEqual(String value) { + return Optional.ofNullable(id.equals(value)).orElse(false); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java new file mode 100644 index 000000000..df529af99 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -0,0 +1,176 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.services.ImportStatusService; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.AppendOverwriteIDValidation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.BrAPICommit; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize.WorkflowInitialization; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.ImportTableProcess; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.ArrayList; +import java.util.Optional; +@Slf4j +@Getter +@Singleton +public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow { + private final ExperimentWorkflowNavigator.Workflow workflow; + private final AppendOverwriteMiddleware importPreviewMiddleware; + private final AppendOverwriteMiddleware brapiCommitMiddleware; + private final ImportStatusService statusService; + + @Inject + public AppendOverwritePhenotypesWorkflow(AppendOverwriteIDValidation expUnitIDValidation, + WorkflowInitialization workflowInitialization, + ImportTableProcess importTableProcess, + BrAPICommit brAPICommit, + ImportStatusService statusService){ + this.statusService = statusService; + this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE; + this.importPreviewMiddleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link( + expUnitIDValidation, + workflowInitialization, + importTableProcess); + this.brapiCommitMiddleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link(brAPICommit); + } + + /** + * Processes the import workflow based on the provided import service context. + * If the provided context is not valid or if the workflow is not equal to the context workflow, returns an empty Optional. + * If the context is null, returns a no-preview result with metadata for this workflow. + * Processes the import preview using middleware, catches and handles any processing errors, and builds the import preview response. + * Updates status based on the processing and commit actions if applicable. + * + * @param context The import service context containing upload, data, program, user, commit flag, and workflow information. + * @return Optional containing ImportWorkflowResult with workflow metadata and import preview response if successful, else empty Optional. + */ + @Override + public Optional process(ImportServiceContext context) { + + // Metadata about this workflow processing the context + ImportWorkflow workflow = ImportWorkflow.builder() + .id(getWorkflow().getId()) + .name(getWorkflow().getName()) + .build(); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .workflow(workflow) // attach metadata of this workflow to response + .importPreviewResponse(Optional.empty()) + .caughtException(Optional.empty()) + .build()); + + // Skip this workflow unless appending or overwriting observation data + if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result with metadata for this workflow + if (context == null) { + return result; + } + + // Build the workflow context for processing the import + ImportContext importContext = ImportContext.builder() + .upload(context.getUpload()) + .importRows(context.getBrAPIImports()) + .data(context.getData()) + .program(context.getProgram()) + .user(context.getUser()) + .commit(context.isCommit()) + .build(); + AppendOverwriteMiddlewareContext workflowContext = AppendOverwriteMiddlewareContext.builder() + .importContext(importContext) + .appendOverwriteWorkflowContext(new AppendOverwriteWorkflowContext()) + .build(); + + // Process the import preview + AppendOverwriteMiddlewareContext processedPreviewContext = this.importPreviewMiddleware.process(workflowContext); + + // Stop and return any errors that occurred while processing + Optional previewException = Optional.ofNullable(processedPreviewContext.getAppendOverwriteWorkflowContext().getProcessError()); + if (previewException.isPresent()) { + log.debug(String.format("%s in %s", previewException.get().getException().getClass().getName(), previewException.get().getLocalTransactionName())); + result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.ofNullable(previewException.get().getException()))); + return result; + } + + // Build and return the preview response + ImportPreviewResponse response = new ImportPreviewResponse(); + response.setStatistics(processedPreviewContext.getAppendOverwriteWorkflowContext().getStatistic().constructPreviewMap()); + response.setRows(new ArrayList<>(processedPreviewContext.getImportContext().getMappedBrAPIImport().values())); + response.setDynamicColumnNames(processedPreviewContext.getImportContext().getUpload().getDynamicColumnNamesList()); + + result.ifPresent(importWorkflowResult -> importWorkflowResult.setImportPreviewResponse(Optional.of(response))); + + log.debug("Finished mapping data to brapi objects"); + statusService.updateMappedData(context.getUpload(), response, "Finished mapping data to brapi objects"); + + if (!context.isCommit()) { + statusService.updateOk(context.getUpload()); + return result; + } else { + + // get total number of new brapi objects to create + long totalObjects = response.getStatistics().values().stream() + .mapToLong(ImportPreviewStatistics::getNewObjectCount) // Extract newObjectCount from each ImportStatistics entry + .sum(); // Sum the newObjectCount values + log.debug("Starting upload to brapi service"); + statusService.startUpload(context.getUpload(), totalObjects, "Starting upload to brapi service"); + log.debug("Creating new objects in brapi service"); + statusService.updateMessage(context.getUpload(), "Creating new objects in brapi service"); + + // Commit the changes from the processed import preview to the BrAPI service + AppendOverwriteMiddlewareContext brapiCommittedContext = this.brapiCommitMiddleware.process(processedPreviewContext); + + Optional brapiCommitException = Optional.ofNullable(brapiCommittedContext.getAppendOverwriteWorkflowContext().getProcessError()); + if (brapiCommitException.isPresent()) { + log.debug(String.format("%s in %s", brapiCommitException.get().getException().getClass()), brapiCommitException.get().getLocalTransactionName()); + result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.ofNullable(brapiCommitException.get().getException()))); + return result; + } + + log.debug("Completed upload to brapi service"); + statusService.finishUpload(context.getUpload(), totalObjects, "Completed upload to brapi service"); + } + + return result; + } + + @Override + public int getOrder() { + return 2; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java new file mode 100644 index 000000000..2fb63fa01 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/BrAPIState.java @@ -0,0 +1,21 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory; + +public interface BrAPIState { +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java new file mode 100644 index 000000000..830de5828 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java @@ -0,0 +1,65 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; + +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import java.util.Optional; + +/** + * Interface representing an action to be performed on the BrAPI service. + * This interface defines two methods: execute() to execute the action on the BrAPI service + * and return any relevant BrAPI state, and getEntity() to get the BrAPI entity being + * acted on based on the provided ExpUnitMiddlewareContext. + * + * @param The type of entity on which the action is being performed. + */ +public interface BrAPIAction { + + /** + * Execute the action on the BrAPI service. + * + * @return An Optional containing the relevant BrAPI state after executing the action. + * @throws ApiException if an error occurs during the execution of the action. + */ + Optional> execute() throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException; + + /** + * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. + * + * @return The ExperimentImportEntity representing the BrAPI entity being acted on. + */ + ExperimentImportEntity getEntity(); +} + + +/* + * Overall module description: + * This BrAPIAction interface defines methods to handle actions on the BrAPI service. + * Developers can implement this interface to execute actions and retrieve the entity being acted upon. + * The execute() method is used to perform actions on the BrAPI service and return the resulting BrAPI state, + * while the getEntity() method retrieves the BrAPI entity based on the provided ExpUnitMiddlewareContext. + * + * The BrAPIAction interface allows for customization of BrAPI service interactions and handling of BrAPI entities. + */ + diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java new file mode 100644 index 000000000..642e46a97 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPICreationFactory.java @@ -0,0 +1,196 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; + +import io.micronaut.context.annotation.Bean; +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Prototype; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.PendingEntityFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.model.ProgramLocation; + +import javax.inject.Inject; + +@Factory +public class BrAPICreationFactory { + private final PendingEntityFactory pendingEntityFactory; + + @Inject + public BrAPICreationFactory(PendingEntityFactory pendingEntityFactory) { + this.pendingEntityFactory = pendingEntityFactory; + } + + /** + * Creates a workflow for creating a BrAPI trial. + * + * @param context The AppendOverwriteMiddlewareContext containing the necessary information for the creation of the trial. + * @param pendingEntityFactory The factory responsible for creating pending entities. + * @return A WorkflowCreation instance for creating a BrAPITrial. + */ + public static WorkflowCreation trialWorkflowCreation(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowCreation(pendingEntityFactory.pendingTrialBean(context)); + } + + /** + * Creates a workflow for creating a BrAPI dataset. + * + * @param context The AppendOverwriteMiddlewareContext containing the necessary information for the creation of the dataset. + * @param pendingEntityFactory The factory responsible for creating pending entities. + * @return A WorkflowCreation instance for creating a BrAPIListDetails. + */ + public static WorkflowCreation datasetWorkflowCreation(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowCreation(pendingEntityFactory.pendingDatasetBean(context)); + } + + /** + * Creates a workflow for creating a BrAPI study. + * + * @param context The AppendOverwriteMiddlewareContext containing the necessary information for the creation of the study. + * @param pendingEntityFactory The factory responsible for creating pending entities. + * @return A WorkflowCreation instance for creating a BrAPIStudy. + */ + public static WorkflowCreation studyWorkflowCreation(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowCreation(pendingEntityFactory.pendingStudyBean(context)); + } + + /** + * Creates a workflow for creating a BrAPI observation. + * + * @param context The AppendOverwriteMiddlewareContext containing the necessary information for the creation of the observation. + * @param pendingEntityFactory The factory responsible for creating pending entities. + * @return A WorkflowCreation instance for creating a BrAPIObservation. + */ + public static WorkflowCreation observationWorkflowCreation(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowCreation(pendingEntityFactory.pendingObservationBean(context)); + } + + /** + * Creates a workflow for creating a BrAPI observation unit. + * + * @param context The AppendOverwriteMiddlewareContext containing the necessary information for the creation of the observation unit. + * @param pendingEntityFactory The factory responsible for creating pending entities. + * @return A WorkflowCreation instance for creating a BrAPIObservationUnit. + */ + public static WorkflowCreation observationUnitWorkflowCreation(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowCreation(pendingEntityFactory.pendingObservationUnitBean(context)); + } + + /** + * Creates a workflow for creating a Program Location. + * + * @param context The AppendOverwriteMiddlewareContext containing the necessary information for the creation of the location. + * @param pendingEntityFactory The factory responsible for creating pending entities. + * @return A WorkflowCreation instance for creating a ProgramLocation. + */ + public static WorkflowCreation locationWorkflowCreation(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowCreation(pendingEntityFactory.pendingLocationBean(context)); + } + + /** + * This method is a Spring bean that creates a prototype instance of a WorkflowCreation object for BrAPITrial entities. + * The WorkflowCreation object is responsible for creating a workflow for a BrAPITrial entity. + * + * @param context The middleware context containing information required for creating the workflow. + * @return A WorkflowCreation object specialized for BrAPITrial entities. + */ + @Bean + @Prototype + public WorkflowCreation trialWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { + return trialWorkflowCreation(context, pendingEntityFactory); + } + + /** + * This method creates a new instance of WorkflowCreation for handling dataset-related tasks within the BrAPIListDetails context. + * The WorkflowCreation instance will be configured as a Prototype bean, meaning it will return a new instance each time it is requested. + * The method takes an AppendOverwriteMiddlewareContext as input, which provides the necessary context for the workflow creation. + * The method returns a WorkflowCreation instance for handling dataset creation operations within the BrAPIListDetails context. + * @param context The AppendOverwriteMiddlewareContext providing the context for the workflow creation process. + * @return A WorkflowCreation instance configured to handle dataset creation operations within the BrAPIListDetails context. + */ + @Bean + @Prototype + public WorkflowCreation datasetWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { + return datasetWorkflowCreation(context, pendingEntityFactory); + } + + /** + * This method creates a bean for creating a workflow for a BrAPI study. + * The bean is of prototype scope, meaning that a new instance of the bean will be created every time it is requested. + * The workflow creation is specific to BrAPIStudy type. + * + * @param context The AppendOverwriteMiddlewareContext object containing the context information for workflow creation. + * @return a WorkflowCreation object specialized for BrAPIStudy, configured using the provided context and pendingEntityFactory. + */ + @Bean + @Prototype + public WorkflowCreation studyWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { + return studyWorkflowCreation(context, pendingEntityFactory); + } + + /** + * This method creates a new instance of WorkflowCreation for BrAPIObservation objects based on the provided AppendOverwriteMiddlewareContext. + * It is marked as a prototype bean, meaning a new instance will be created each time it is injected. + * + * @param context The AppendOverwriteMiddlewareContext containing the necessary information for the workflow creation. + * @return A WorkflowCreation instance for BrAPIObservation objects. + */ + @Bean + @Prototype + public WorkflowCreation observationWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { + return observationWorkflowCreation(context, pendingEntityFactory); + } + + /** + * This method is responsible for creating a new instance of WorkflowCreation for BrAPIObservationUnit. + * It leverages the observationUnitWorkflowCreation method to initialize the WorkflowCreation object. + * + * @param context the AppendOverwriteMiddlewareContext object containing relevant context information + * @return a new instance of WorkflowCreation object + * + */ + @Bean + @Prototype + public WorkflowCreation observationUnitWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { + return observationUnitWorkflowCreation(context, pendingEntityFactory); + } + + /** + * This method creates a new WorkflowCreation instance for handling ProgramLocation objects by appending or overwriting the entities. + * The WorkflowCreation instance is specific to each invocation. + * + * @param context the AppendOverwriteMiddlewareContext object containing the context for workflow creation + * @return a WorkflowCreation instance configured for handling ProgramLocation objects + */ + @Bean + @Prototype + public WorkflowCreation locationWorkflowCreationBean(AppendOverwriteMiddlewareContext context) { + return locationWorkflowCreation(context, pendingEntityFactory); + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java new file mode 100644 index 000000000..dabd5811c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIReadFactory.java @@ -0,0 +1,108 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; + +import io.micronaut.context.annotation.Bean; +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Prototype; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.PendingEntityFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.model.ProgramLocation; + +import javax.inject.Inject; + +@Factory +public class BrAPIReadFactory { + private final PendingEntityFactory pendingEntityFactory; + + @Inject + public BrAPIReadFactory(PendingEntityFactory pendingEntityFactory) { + this.pendingEntityFactory = pendingEntityFactory; + } + + public static WorkflowReadInitialization trialWorkflowReadInitialization(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowReadInitialization(pendingEntityFactory.pendingTrialBean(context)); + } + + public static WorkflowReadInitialization observationUnitWorkflowReadInitialization(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowReadInitialization(pendingEntityFactory.pendingObservationUnitBean(context)); + } + + public static WorkflowReadInitialization germplasmWorkflowReadInitialization(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowReadInitialization(pendingEntityFactory.pendingGermplasmBean(context)); + } + + public static WorkflowReadInitialization datasetWorkflowReadInitialization(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowReadInitialization(pendingEntityFactory.pendingDatasetBean(context)); + } + + public static WorkflowReadInitialization studyWorkflowReadInitialization(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowReadInitialization(pendingEntityFactory.pendingStudyBean(context)); + } + + public static WorkflowReadInitialization locationWorkflowReadInitialization(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowReadInitialization(pendingEntityFactory.pendingLocationBean(context)); + } + + @Bean + @Prototype + public WorkflowReadInitialization trialWorkflowReadInitializationBean(AppendOverwriteMiddlewareContext context) { + return trialWorkflowReadInitialization(context, pendingEntityFactory); + } + + @Bean + @Prototype + public WorkflowReadInitialization observationUnitWorkflowReadInitializationBean(AppendOverwriteMiddlewareContext context) { + return observationUnitWorkflowReadInitialization(context, pendingEntityFactory); + } + + @Bean + @Prototype + public WorkflowReadInitialization germplasmWorkflowReadInitializationBean(AppendOverwriteMiddlewareContext context) { + return germplasmWorkflowReadInitialization(context, pendingEntityFactory); + } + + @Bean + @Prototype + public WorkflowReadInitialization datasetWorkflowReadInitializationBean(AppendOverwriteMiddlewareContext context) { + return datasetWorkflowReadInitialization(context, pendingEntityFactory); + } + + @Bean + @Prototype + public WorkflowReadInitialization studyWorkflowReadInitializationBean(AppendOverwriteMiddlewareContext context) { + return studyWorkflowReadInitialization(context, pendingEntityFactory); + } + + @Bean + @Prototype + public WorkflowReadInitialization locationWorkflowReadInitializationBean(AppendOverwriteMiddlewareContext context) { + return locationWorkflowReadInitialization(context, pendingEntityFactory); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java new file mode 100644 index 000000000..cb73627b1 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIUpdateFactory.java @@ -0,0 +1,139 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; + +import io.micronaut.context.annotation.Bean; +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.PendingEntityFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; + +import javax.inject.Inject; +import java.util.List; +import java.util.Optional; + +@Factory +public class BrAPIUpdateFactory { + private final PendingEntityFactory pendingEntityFactory; + + @Inject + public BrAPIUpdateFactory(PendingEntityFactory pendingEntityFactory) { + this.pendingEntityFactory = pendingEntityFactory; + } + + private static WorkflowUpdate trialWorkflowUpdate(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowUpdate<>(pendingEntityFactory.pendingTrialBean(context)); + } + + private static WorkflowUpdate observationWorkflowUpdate(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowUpdate<>(pendingEntityFactory.pendingObservationBean(context)); + } + + private static WorkflowUpdate datasetWorkflowUpdate(AppendOverwriteMiddlewareContext context, + PendingEntityFactory pendingEntityFactory) { + return new WorkflowUpdate<>(pendingEntityFactory.pendingDatasetBean(context)); + } + + @Bean + @Prototype + public WorkflowUpdate trialWorkflowUpdateBean(AppendOverwriteMiddlewareContext context) { + return trialWorkflowUpdate(context, pendingEntityFactory); + } + + @Bean + @Prototype + public WorkflowUpdate observationWorkflowUpdateBean(AppendOverwriteMiddlewareContext context) { + return observationWorkflowUpdate(context, pendingEntityFactory); + } + + @Bean + @Prototype + public WorkflowUpdate datasetWorkflowUpdateBean(AppendOverwriteMiddlewareContext context) { + return datasetWorkflowUpdate(context, pendingEntityFactory); + } + + @Slf4j + @Prototype + public static class WorkflowUpdate implements BrAPIAction { + private final ExperimentImportEntity entity; + + private WorkflowUpdate(ExperimentImportEntity entity) { + + this.entity = entity; + } + + public Optional> execute() throws ApiException { + return saveAndUpdateCache(entity.copyWorkflowMembers(ImportObjectState.MUTATED)); + } + + /** + * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. + * + * @return The ExperimentImportEntity representing the BrAPI entity being acted on. + */ + @Override + public ExperimentImportEntity getEntity() { + return null; + } + + public Optional> getBrAPIState() { + try { + return Optional.of(new BrAPIUpdateState(entity.getBrAPIState(ImportObjectState.MUTATED))); + } catch (ApiException e) { + // TODO: add specific error messages to entity service + log.error("Error getting..."); + throw new InternalServerException("Error getting...", e); + } + + } + protected Optional> saveAndUpdateCache(List members) throws IllegalArgumentException, ApiException { + + if (members == null) { + throw new IllegalArgumentException("BrAPI entity cannot be null"); + } + List savedMembers = entity.brapiPut(members); + entity.updateWorkflow(savedMembers); + return Optional.of(new BrAPIUpdateState(savedMembers)); + } + + + @Getter + public class BrAPIUpdateState implements BrAPIState { + private final List members; + + public BrAPIUpdateState(List existingMembers) { this.members = existingMembers; } + + public boolean restore() throws ApiException { + return saveAndUpdateCache(this.getMembers()).isPresent(); + } + } + + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java new file mode 100644 index 000000000..c5f58b852 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowCreation.java @@ -0,0 +1,106 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; + +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import java.util.List; +import java.util.Optional; + +@Slf4j +@Prototype +public class WorkflowCreation implements BrAPIAction { + + private final ExperimentImportEntity entity; + + + protected WorkflowCreation(ExperimentImportEntity entity) { + this.entity = entity; + } + + + + /** + * Executes the creation process for entities. + * @return an Optional containing the BrAPI state after execution + * @throws ApiException if an error occurs during execution + */ + public Optional> execute() throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + List newMembers = entity.copyWorkflowMembers(ImportObjectState.NEW); + try { + List createdMembers = entity.brapiPost(newMembers); + entity.updateWorkflow(createdMembers); + return Optional.of(new BrAPICreationState<>(createdMembers)); + } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { + log.error("Error creating..."); + throw e; + } + } + + /** + * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. + * + * @return The ExperimentImportEntity representing the BrAPI entity being acted on. + */ + @Override + public ExperimentImportEntity getEntity() { + return null; + } + + /** + * Inner class representing the state of creation for BrAPI entities. + * @param the type of entity + */ + @Getter + public class BrAPICreationState implements BrAPIState { + + private final List members; + + /** + * Constructor for BrAPICreationState class. + * @param createdMembers the list of created members + */ + public BrAPICreationState(List createdMembers) { + this.members = createdMembers; + } + + /** + * Undo the creation operation by deleting the created members. + * @return true if undo operation is successful, false otherwise + */ + public boolean undo() { + List createdMembers = this.getMembers(); + try { + return entity.brapiDelete(createdMembers); + } catch (ApiException e) { + log.error("Error deleting..."); + throw new InternalServerException("Error deleting...", e); + } + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java new file mode 100644 index 000000000..717a78f97 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java @@ -0,0 +1,85 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action; + +import io.micronaut.context.annotation.Prototype; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity; +import org.breedinginsight.utilities.Utilities; + +import java.util.List; +import java.util.Optional; + +@Slf4j +@Prototype +public class WorkflowReadInitialization implements BrAPIAction { + private final ExperimentImportEntity entity; // The entity used for read operations initialization + + protected WorkflowReadInitialization(ExperimentImportEntity entity) { + this.entity = entity; + } + + + /** + * Executes the read workflow by fetching members from the entity and initializing the workflow. + * + * @return an Optional containing the BrAPIState representing the completed read workflow + * @throws ApiException if an error occurs during execution + */ + public Optional> execute() throws ApiException { + try { + List fetchedMembers = entity.brapiRead(); + entity.initializeWorkflow(fetchedMembers); + return Optional.of(new WorkflowReadInitialization.BrAPIReadState(fetchedMembers)); + } catch(ApiException e) { + log.error(String.format("Error fetching %s: %s", entity.getClass().getName(), Utilities.generateApiExceptionLogMessage(e)), e); + throw new ApiException(e); + } + } + + /** + * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. + * + * @return The ExperimentImportEntity representing the BrAPI entity being acted on. + */ + @Override + public ExperimentImportEntity getEntity() { + return null; + } + + /** + * The state class representing the result of a read operation. + * + * @param the type of entity members contained in the state + */ + @Getter + public static class BrAPIReadState implements BrAPIState { + + private final List members; // The list of members fetched during the read operation + + /** + * Constructs a new BrAPIReadState object with the provided list of members. + * + * @param fetchedMembers the list of members fetched during the read operation + */ + public BrAPIReadState(List fetchedMembers) { this.members = fetchedMembers; } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/EmptyData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/EmptyData.java new file mode 100644 index 000000000..6c0df56ad --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/EmptyData.java @@ -0,0 +1,129 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; + +import io.micronaut.context.annotation.Prototype; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.model.User; +import org.breedinginsight.utilities.Utilities; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Prototype +public class EmptyData extends VisitedObservationData { + String brapiReferenceSource; + boolean isCommit; + String germplasmName; + BrAPIStudy study; + String phenoColumnName; + UUID trialId; + UUID studyId; + UUID unitId; + String studyYear; + BrAPIObservationUnit observationUnit; + User user; + Program program; + private final StudyService studyService; + private final ObservationService observationService; + + public EmptyData(String brapiReferenceSource, + boolean isCommit, + String germplasmName, + BrAPIStudy study, + String phenoColumnName, + UUID trialId, + UUID studyId, + UUID unitId, + String studyYear, + BrAPIObservationUnit observationUnit, + User user, + Program program, + StudyService studyService, + ObservationService observationService) { + this.brapiReferenceSource = brapiReferenceSource; + this.isCommit = isCommit; + this.germplasmName = germplasmName; + this.study = study; + this.phenoColumnName = phenoColumnName; + this.trialId = trialId; + this.studyId = studyId; + this.unitId = unitId; + this.studyYear = studyYear; + this.observationUnit = observationUnit; + this.user = user; + this.program = program; + this.studyService = studyService; + this.observationService = observationService; + } + + @Override + public Optional> getValidationErrors() { + return Optional.empty(); + } + + @Override + public PendingImportObject constructPendingObservation() { + /** + * TODO: fix the front end experiment import preview table so that it won't break if a table row has + * an empty observations array when there are phenotype columns. Once this is fixed on the front end, + * delete the work-around below and simply have this method return null. + */ + + String seasonDbId = studyService.yearToSeasonDbIdFromDatabase(studyYear, program.getId()); + + // Generate a new ID for the observation + UUID observationId = UUID.randomUUID(); + + // Construct the new observation + BrAPIObservation newObservation = observationService.constructNewBrAPIObservation(isCommit, + germplasmName, + phenoColumnName, + study, + seasonDbId, + observationUnit, + "", // the value of the observation is empty + trialId, + studyId, + unitId, + observationId, + brapiReferenceSource, + user, + program); + + // Construct a pending observation with a status set to NEW + return new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); + } + + @Override + public void updateTally(AppendStatistic statistic) { + + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java new file mode 100644 index 000000000..1b14a12a4 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/InitialData.java @@ -0,0 +1,163 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; + +import com.google.gson.Gson; +import io.micronaut.context.annotation.Prototype; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.model.User; +import org.breedinginsight.utilities.Utilities; + +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Prototype +public class InitialData extends VisitedObservationData { + String brapiReferenceSource; + boolean isCommit; + String germplasmName; + BrAPIStudy study; + String cellData; + String timestamp; + String phenoColumnName; + String timestampColumnName; + Trait trait; + ExperimentObservation row; + UUID trialId; + UUID studyId; + UUID unitId; + String studyYear; + BrAPIObservationUnit observationUnit; + User user; + Program program; + private final FieldValidator fieldValidator; + private final StudyService studyService; + private final ObservationService observationService; + Gson gson; + + InitialData(String brapiReferenceSource, + boolean isCommit, + String germplasmName, + BrAPIStudy study, + String cellData, + String timestamp, + String phenoColumnName, + String timestampColumnName, + Trait trait, + ExperimentObservation row, + UUID trialId, + UUID studyId, + UUID unitId, + String studyYear, + BrAPIObservationUnit observationUnit, + User user, + Program program, + FieldValidator fieldValidator, + StudyService studyService, + ObservationService observationService) { + this.brapiReferenceSource = brapiReferenceSource; + this.isCommit = isCommit; + this.germplasmName = germplasmName; + this.study = study; + this.cellData = cellData; + this.timestamp = timestamp; + this.phenoColumnName = phenoColumnName; + this.timestampColumnName = timestampColumnName; + this.trait = trait; + this.row = row; + this.trialId = trialId; + this.studyId = studyId; + this.unitId = unitId; + this.studyYear = studyYear; + this.observationUnit = observationUnit; + this.user = user; + this.program = program; + this.fieldValidator = fieldValidator; + this.studyService = studyService; + this.observationService = observationService; + this.gson = new Gson(); + } + @Override + public Optional> getValidationErrors() { + List errors = new ArrayList<>(); + + // Validate observation value + fieldValidator.validateField(phenoColumnName, cellData, trait).ifPresent(errors::add); + + // Validate timestamp + fieldValidator.validateField(timestampColumnName, timestamp, null).ifPresent(errors::add); + + return Optional.ofNullable(errors.isEmpty() ? null : errors); + } + + @Override + public PendingImportObject constructPendingObservation() { + String seasonDbId = studyService.yearToSeasonDbIdFromDatabase(studyYear, program.getId()); + + // Generate a new ID for the observation + UUID observationId = UUID.randomUUID(); + + // Construct the new observation + BrAPIObservation newObservation = observationService.constructNewBrAPIObservation(isCommit, + germplasmName, + phenoColumnName, + study, + seasonDbId, + observationUnit, + cellData, + trialId, + studyId, + unitId, + observationId, + brapiReferenceSource, + user, + program); + + // Add a timestamp if included + if (timestamp != null && !timestamp.isBlank()) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(observationService.parseDateTime(timestamp)); + newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + } + + // Construct a pending observation with a status set to NEW + return new PendingImportObject<>(ImportObjectState.NEW, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); + + } + + @Override + public void updateTally(AppendStatistic statistic) { + statistic.incrementNewCount(1); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java new file mode 100644 index 000000000..78bd51a30 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java @@ -0,0 +1,189 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.HttpStatus; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Prototype +public class OverwrittenData extends VisitedObservationData { + + FieldValidator fieldValidator; + ObservationService observationService; + Gson gson; + boolean canOverwrite; + boolean isCommit; + String unitId; + Trait trait; + String phenoColumnName; + String timestampColumnName; + String cellData; + String timestamp; + String reason; + BrAPIObservation observation; + UUID userId; + Program program; + + @Inject + public OverwrittenData(boolean canOverwrite, + boolean isCommit, + String unitId, + Trait trait, + String phenoColumnName, + String timestampColumnName, + String cellData, + String timestamp, + String reason, + BrAPIObservation observation, + UUID userId, + Program program, + FieldValidator fieldValidator, + ObservationService observationService) { + this.canOverwrite = canOverwrite; + this.isCommit = isCommit; + this.unitId = unitId; + this.trait = trait; + this.phenoColumnName = phenoColumnName; + this.timestampColumnName = timestampColumnName; + this.cellData = cellData; + this.timestamp = timestamp; + this.reason = reason; + this.observation = observation; + this.userId = userId; + this.program = program; + this.fieldValidator = fieldValidator; + this.observationService = observationService; + this.gson = new Gson(); + } + + @Override + public Optional> getValidationErrors() { + List errors = new ArrayList<>(); + + // Errors for trying to change protected data + if (!canOverwrite) { + if (!isValueMatched()) { + errors.add(new ValidationError(phenoColumnName, String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", unitId, phenoColumnName), HttpStatus.UNPROCESSABLE_ENTITY)); + } + if (!isTimestampMatched()) { + errors.add(new ValidationError(timestampColumnName, String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", unitId, timestampColumnName), HttpStatus.UNPROCESSABLE_ENTITY)); + } + } + + // Validate observation value + fieldValidator.validateField(phenoColumnName, cellData, trait).ifPresent(errors::add); + + // Validate timestamp + fieldValidator.validateField(timestampColumnName, timestamp, null).ifPresent(errors::add); + + return Optional.ofNullable(errors.isEmpty() ? null : errors); + } + + @Override + public PendingImportObject constructPendingObservation() { + // Construct a pending observation with a status set to MUTATED + PendingImportObject pendingUpdatedObservation = new PendingImportObject<>(ImportObjectState.MUTATED, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); + BrAPIObservation update = pendingUpdatedObservation.getBrAPIObject(); + String original = null; + + if (!isValueMatched()) { + // Update the observation value + update.setValue(cellData); + + // Record original observation value for changelog entry + original = observation.getValue(); + } + + if (!isTimestampMatched()) { + // Update the timestamp + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(observationService.parseDateTime(timestamp)); + update.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + + // Add original timestamp to changelog entry + original = Optional.ofNullable(original).map(o -> o + " " + observation.getObservationTimeStamp()).orElse(String.valueOf(observation.getObservationTimeStamp())); + } + + // If the change is to be committed, attach a record of the change as BrAPI observation additional info + if (isCommit) { + // Create the changelog field in observation additional info if it does not already exist + createAdditionalInfoChangeLog(update); + + // Construct a changelog entry + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String rightNow = formatter.format(OffsetDateTime.now()); + ChangeLogEntry entry = new ChangeLogEntry(original, Optional.ofNullable(reason).orElse(""), userId, rightNow); + + // Add the entry to the changelog + update.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(entry).getAsJsonObject()); + } + + return pendingUpdatedObservation; + } + + @Override + public void updateTally(AppendStatistic statistic) { + statistic.incrementMutatedCount(1); + } + + private void createAdditionalInfoChangeLog(BrAPIObservation update) { + if (update.getAdditionalInfo().isJsonNull()) { + update.setAdditionalInfo(new JsonObject()); + update.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + if (update.getAdditionalInfo() != null && !update.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { + update.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + } + + private boolean isValueMatched() { + return cellData.equals(observation.getValue()); + } + + private boolean isTimestampMatched() { + if (timestamp == null) { + return observation.getObservationTimeStamp() == null; + } else { + return observationService.parseDateTime(timestamp).equals(observation.getObservationTimeStamp()); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java new file mode 100644 index 000000000..84f8bcf75 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java @@ -0,0 +1,177 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; + +import io.micronaut.context.annotation.Bean; +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Prototype; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.model.User; + +import javax.inject.Inject; +import java.util.UUID; + + +@Factory +public class ProcessedDataFactory { + private final FieldValidator fieldValidator; + private final StudyService studyService; + private final ObservationService observationService; + + @Inject + public ProcessedDataFactory(FieldValidator fieldValidator, + StudyService studyService, + ObservationService observationService) { + + this.fieldValidator = fieldValidator; + this.studyService = studyService; + this.observationService = observationService; + } + + public static InitialData initialData(String brapiReferenceSource, + boolean isCommit, + String germplasmName, + BrAPIStudy study, + String cellData, + String timestamp, + String phenoColumnName, + String timestampColumnName, + Trait trait, + ExperimentObservation row, + UUID trialId, + UUID studyId, + UUID unitId, + String studyYear, + BrAPIObservationUnit observationUnit, + User user, + Program program, + FieldValidator fieldValidator, + StudyService studyService, + ObservationService observationService) { + return new InitialData(brapiReferenceSource, isCommit, germplasmName, study, cellData, timestamp, phenoColumnName, timestampColumnName, trait, row, trialId, studyId, unitId, studyYear, observationUnit, user, program, fieldValidator, studyService, observationService); + } + + public static OverwrittenData overwrittenData(boolean canOverwrite, + boolean isCommit, + String unitId, + Trait trait, + String phenoColumnName, + String timestampColumnName, + String cellData, + String timestamp, + String reason, + BrAPIObservation observation, + UUID userId, + Program program, + FieldValidator fieldValidator, + ObservationService observationService) { + return new OverwrittenData(canOverwrite, isCommit, unitId, trait, phenoColumnName, timestampColumnName, cellData, timestamp, reason, observation, userId, program, fieldValidator, observationService); + } + + public static UnchangedData unchangedData(BrAPIObservation observation, Program program) { + return new UnchangedData(observation, program); + } + + public static EmptyData emptyData(String brapiReferenceSource, + boolean isCommit, + String germplasmName, + BrAPIStudy study, + String phenoColumnName, + UUID trialId, + UUID studyId, + UUID unitId, + String studyYear, + BrAPIObservationUnit observationUnit, + User user, + Program program, + StudyService studyService, + ObservationService observationService) { + return new EmptyData(brapiReferenceSource, isCommit, germplasmName, study, phenoColumnName, trialId, studyId, unitId, studyYear, observationUnit, user, program, studyService, observationService); + } + + @Bean + @Prototype + public InitialData initialDataBean(String brapiReferenceSource, + boolean isCommit, + String germplasmName, + BrAPIStudy study, + String cellData, + String timestamp, + String phenoColumnName, + String timestampColumnName, + Trait trait, + ExperimentObservation row, + UUID trialId, + UUID studyId, + UUID unitId, + String studyYear, + BrAPIObservationUnit observationUnit, + User user, + Program program) { + return initialData(brapiReferenceSource, isCommit, germplasmName, study, cellData, timestamp, phenoColumnName, timestampColumnName, trait, row, trialId, studyId, unitId, studyYear, observationUnit, user, program, fieldValidator, studyService, observationService); + } + + @Bean + @Prototype + public OverwrittenData overwrittenDataBean(boolean canOverwrite, + boolean isCommit, + String unitId, + Trait trait, + String phenoColumnName, + String timestampColumnName, + String cellData, + String timestamp, + String reason, + BrAPIObservation observation, + UUID userId, + Program program) { + return overwrittenData(canOverwrite, isCommit, unitId, trait, phenoColumnName, timestampColumnName, cellData, timestamp, reason, observation, userId, program, fieldValidator, observationService); + } + + @Bean + @Prototype + public UnchangedData unchangedDataBean(BrAPIObservation observation, Program program) { + return unchangedData(observation, program); + } + + @Bean + @Prototype + public EmptyData emptyDataBean(String brapiReferenceSource, + boolean isCommit, + String germplasmName, + BrAPIStudy study, + String phenoColumnName, + UUID trialId, + UUID studyId, + UUID unitId, + String studyYear, + BrAPIObservationUnit observationUnit, + User user, + Program program) { + return emptyData(brapiReferenceSource, isCommit, germplasmName, study, phenoColumnName, trialId, studyId, unitId, studyYear, observationUnit, user, program, studyService, observationService); + } +} + diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UnchangedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UnchangedData.java new file mode 100644 index 000000000..4b1e6b92f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UnchangedData.java @@ -0,0 +1,60 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; + +import io.micronaut.context.annotation.Prototype; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.List; +import java.util.Optional; + +@Prototype +public class UnchangedData extends VisitedObservationData { + BrAPIObservation observation; + Program program; + + @Inject + public UnchangedData(BrAPIObservation observation, Program program) { + this.observation = observation; + this.program = program; + } + @Override + public Optional> getValidationErrors() { + return Optional.empty(); + } + + @Override + public PendingImportObject constructPendingObservation() { + // Construct a pending observation with a status set to EXISTING + PendingImportObject pendingExistingObservation = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); + + return pendingExistingObservation; + } + + @Override + public void updateTally(AppendStatistic statistic) { + statistic.incrementExistingCount(1); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/VisitedObservationData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/VisitedObservationData.java new file mode 100644 index 000000000..32a1b3d0b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/VisitedObservationData.java @@ -0,0 +1,32 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; + +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; + +import java.util.List; +import java.util.Optional; + +public abstract class VisitedObservationData { + abstract public Optional> getValidationErrors(); + abstract public PendingImportObject constructPendingObservation(); + abstract public void updateTally(AppendStatistic statistic); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java new file mode 100644 index 000000000..c98d9cf83 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java @@ -0,0 +1,95 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; + +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import java.util.List; + +/** + * Interface for importing entities related to experiments using BrAPI service. + */ +public interface ExperimentImportEntity { + + /** + * Create new objects generated by the workflow in the BrAPI service. + * @param members List of entities to be created + * @return List of created entities + * @throws ApiException if there is an issue with the API call + */ + public List brapiPost(List members) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException; + + /** + * Fetch objects required by the workflow from the BrAPI service. + * @return List of fetched entities + * @throws ApiException if there is an issue with the API call + */ + public List brapiRead() throws ApiException; + + /** + * Commit objects changed by the workflow to the BrAPI service. + * @param members List of entities to be updated + * @param Type of entities + * @return List of updated entities + * @throws ApiException if there is an issue with the API call + * @throws IllegalArgumentException if method arguments are invalid + */ + public List brapiPut(List members) throws ApiException, IllegalArgumentException; + + /** + * Remove objects created by the workflow from the BrAPI service. + * @param members List of entities to be deleted + * @param Type of entities + * @return true if deletion is successful, false otherwise + * @throws ApiException if there is an issue with the API call + */ + public boolean brapiDelete(List members) throws ApiException; + + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the BrAPI service. + * @param status State of the objects + * @return List of deep copies of entities + * @throws ApiException if there is an issue with the API call + */ + public List getBrAPIState(ImportObjectState status) throws ApiException; + + /** + * For workflow pending import objects of a given state, construct deep copies of the objects from the workflow context. + * @param status State of the objects + * @return List of deep copies of entities from workflow context + */ + public List copyWorkflowMembers(ImportObjectState status); + + /** + * For objects in the workflow context, update any foreign-key fields with values generated by the BrAPI service. + * @param members List of entities to be updated + * @param Type of entities + */ + public void updateWorkflow(List members); + + /** + * Populate the workflow context with objects needed by the workflow. + * @param members List of entities to be initialized + * @param Type of entities + */ + public void initializeWorkflow(List members); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java new file mode 100644 index 000000000..3e8dd03e5 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java @@ -0,0 +1,249 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; + +import io.micronaut.context.annotation.Prototype; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.request.BrAPIListNewRequest; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Prototype +public class PendingDataset implements ExperimentImportEntity { + AppendOverwriteWorkflowContext cache; + ImportContext importContext; + BrAPIListDAO brAPIListDAO; + DatasetService datasetService; + ExperimentUtilities experimentUtilities; + + public PendingDataset(AppendOverwriteMiddlewareContext context, + BrAPIListDAO brAPIListDAO, + DatasetService datasetService, + ExperimentUtilities experimentUtilities) { + this.cache = context.getAppendOverwriteWorkflowContext(); + this.importContext = context.getImportContext(); + this.brAPIListDAO = brAPIListDAO; + this.datasetService = datasetService; + this.experimentUtilities = experimentUtilities; + } + /** + * Create new objects generated by the workflow in the BrAPI service. + * + * @param members List of entities to be created + * @return List of created entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiPost(List members) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + // Construct BrAPI list requests + List requests = members.stream().map(details -> { + BrAPIListNewRequest request = new BrAPIListNewRequest(); + request.setListName(details.getListName()); + request.setListType(details.getListType()); + request.setExternalReferences(details.getExternalReferences()); + request.setAdditionalInfo(details.getAdditionalInfo()); + request.data(details.getData()); + return request; + }).collect(Collectors.toList()); + + // The BrAPI service returns summaries with no data details but with system-generated dbIds + List summaries = brAPIListDAO.createBrAPILists(requests, importContext.getProgram().getId(), importContext.getUpload()); + + // Return the dataset data with system-generated dbId + for (BrAPIListSummary summary : summaries) { + for (BrAPIListDetails member : members) { + if (member.getListName().equals(summary.getListName())) { + member.setListDbId(summary.getListDbId()); + } + } + } + + return members; + } + + /** + * Fetch objects required by the workflow from the BrAPI service. + * + * @return List of fetched entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiRead() throws ApiException { + // Get the id of the dataset belonging to the required exp units + String datasetId = cache.getTrialByNameNoScope().values().iterator().next().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + + // Get the dataset belonging to required exp units + return List.of(datasetService.fetchDatasetById(datasetId, importContext.getProgram()).orElseThrow(ApiException::new)); + } + + /** + * Commit objects changed by the workflow to the BrAPI service. + * + * @param members List of entities to be updated + * @return List of updated entities + * @throws ApiException if there is an issue with the API call + * @throws IllegalArgumentException if method arguments are invalid + */ + @Override + public List brapiPut(List members) throws ApiException, IllegalArgumentException { + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIListDetails.class)) { + return new ArrayList(); + } + + List updatedDatasets = new ArrayList<>(); + for (U member : members) { + BrAPIListDetails obsVarList = (BrAPIListDetails) member; + String obsVarListDbId = obsVarList.getListDbId(); + + // Get the current observation variables for the dataset from the BrAPI service + List existingObsVarIds = brAPIListDAO.getListById(obsVarListDbId, importContext.getProgram().getId()).getResult().getData(); + + // Find any observation variables that need to be added to the list in the BrAPI service + List newObsVarIds = obsVarList + .getData() + .stream() + .filter(obsVarId -> !existingObsVarIds.contains(obsVarId)).collect(Collectors.toList()); + + // Save the additions to the list in the BrAPI service + List obsVarIds = new ArrayList<>(existingObsVarIds); + obsVarIds.addAll(newObsVarIds); + obsVarList.setData(obsVarIds); + brAPIListDAO.updateBrAPIList(obsVarListDbId, obsVarList, importContext.getProgram().getId()); + } + + return updatedDatasets; + } + + /** + * Remove objects created by the workflow from the BrAPI service. + * + * @param members List of entities to be deleted + * @return true if deletion is successful, false otherwise + * @throws ApiException if there is an issue with the API call + */ + @Override + public boolean brapiDelete(List members) throws ApiException { + // TODO: implement delete list for BrAPIJavaTestServer + return false; + } + + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the BrAPI service. + * + * @param status State of the objects + * @return List of deep copies of entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List getBrAPIState(ImportObjectState status) throws ApiException { + return new ArrayList<>(); + } + + /** + * For workflow pending import objects of a given state, construct deep copies of the objects from the workflow context. + * + * @param status State of the objects + * @return List of deep copies of entities from workflow context + */ + @Override + public List copyWorkflowMembers(ImportObjectState status) { + return experimentUtilities.copyWorkflowCachePendingBrAPIObjects(cache.getObsVarDatasetByName(), BrAPIListDetails.class, status); + } + + /** + * For objects in the workflow context, update any foreign-key fields with values generated by the BrAPI service. + * + * @param members List of entities to be updated + */ + @Override + public void updateWorkflow(List members) { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIListDetails.class)) { + return; + } + + for (U member : members) { + BrAPIListDetails dataset = (BrAPIListDetails) member; + + // Update the dataset dbId + cache.getObsVarDatasetByName().get(dataset.getListName()).getBrAPIObject().setListDbId(dataset.getListDbId()); + } + + } + + /** + * Populate the workflow context with objects needed by the workflow. + * + * @param members List of entities to be initialized + */ + @Override + public void initializeWorkflow(List members) { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIListDetails.class)) { + return; + } + + // Construct the pending dataset from the BrAPI observation variable list + List> pendingDatasets = members.stream() + .map(m -> (BrAPIListDetails) m) + .map(dataset -> datasetService.constructPIOFromDataset(dataset, importContext.getProgram())) + .collect(Collectors.toList()); + + // Construct a hashmap to look up the pending dataset by dataset name + Map> pendingDatasetByName = pendingDatasets.stream() + .collect(Collectors.toMap(pio -> pio.getBrAPIObject().getListName(),pio -> pio)); + + // Construct a hashmap to look up the pending dataset by the observation unit ID of a unit stored in the BrAPI service + Map> pendingObsDatasetByOUId = cache.getPendingObsUnitByOUId().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> { + if (cache.getPendingTrialByOUId().isEmpty() || + pendingDatasetByName.isEmpty() || + !cache.getPendingTrialByOUId().values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + throw new IllegalStateException("There is not an observation data set for this unit: " + e.getKey()); + } + return pendingDatasetByName.values().iterator().next(); + } + )); + + // Add the maps to the context for use in processing import + cache.setObsVarDatasetByName(pendingDatasetByName); + cache.setPendingObsDatasetByOUId(pendingObsDatasetByOUId); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java new file mode 100644 index 000000000..20e827f2f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java @@ -0,0 +1,167 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; + +import io.micronaut.context.annotation.Bean; +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapi.v2.dao.*; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.*; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.ProgramLocationService; + +import javax.inject.Inject; + +@Factory +public class PendingEntityFactory { + private final TrialService trialService; + private final BrAPITrialDAO brapiTrialDAO; + private final BrAPIObservationUnitDAO observationUnitDAO; + private final ObservationUnitService observationUnitService; + private final StudyService studyService; + private final BrAPIStudyDAO brAPIStudyDAO; + private final GermplasmService germplasmService; + private final BrAPIListDAO brAPIListDAO; + private final DatasetService datasetService; + private final BrAPIObservationDAO brAPIObservationDAO; + private final OntologyService ontologyService; + private final ProgramLocationService programLocationService; + private final LocationService locationService; + private final ExperimentUtilities experimentUtilities; + + @Inject + public PendingEntityFactory(TrialService trialService, + BrAPITrialDAO brapiTrialDAO, + BrAPIObservationUnitDAO observationUnitDAO, + ObservationUnitService observationUnitService, + StudyService studyService, + BrAPIStudyDAO brAPIStudyDAO, + GermplasmService germplasmService, + BrAPIListDAO brAPIListDAO, + DatasetService datasetService, + BrAPIObservationDAO brAPIObservationDAO, + OntologyService ontologyService, ProgramLocationService programLocationService, LocationService locationService, + ExperimentUtilities experimentUtilities) { + this.trialService = trialService; + this.brapiTrialDAO = brapiTrialDAO; + this.observationUnitDAO = observationUnitDAO; + this.observationUnitService = observationUnitService; + this.studyService = studyService; + this.brAPIStudyDAO = brAPIStudyDAO; + this.germplasmService = germplasmService; + this.brAPIListDAO = brAPIListDAO; + this.datasetService = datasetService; + this.brAPIObservationDAO = brAPIObservationDAO; + this.ontologyService = ontologyService; + this.programLocationService = programLocationService; + this.locationService = locationService; + this.experimentUtilities = experimentUtilities; + } + + public static PendingTrial pendingTrial(AppendOverwriteMiddlewareContext context, + TrialService trialService, + BrAPITrialDAO brapiTrialDAO, + ExperimentUtilities experimentUtilities) { + return new PendingTrial(context, trialService, brapiTrialDAO, experimentUtilities); + } + + public static PendingObservationUnit pendingObservationUnit(AppendOverwriteMiddlewareContext context, + BrAPIObservationUnitDAO observationUnitDAO, + ObservationUnitService observationUnitService, + ExperimentUtilities experimentUtilities) { + return new PendingObservationUnit(context, observationUnitDAO, observationUnitService, experimentUtilities); + } + + public static PendingStudy pendingStudy(AppendOverwriteMiddlewareContext context, + StudyService studyService, + BrAPIStudyDAO brAPIStudyDAO, + ExperimentUtilities experimentUtilities) { + return new PendingStudy(context, studyService, brAPIStudyDAO, experimentUtilities); + } + + public static PendingGermplasm pendingGermplasm(AppendOverwriteMiddlewareContext context, + GermplasmService germplasmService, + ExperimentUtilities experimentUtilities) { + return new PendingGermplasm(context, germplasmService, experimentUtilities); + } + + public static PendingDataset pendingDataset(AppendOverwriteMiddlewareContext context, + BrAPIListDAO brAPIListDAO, + DatasetService datasetService, + ExperimentUtilities experimentUtilities) { + return new PendingDataset(context, brAPIListDAO, datasetService, experimentUtilities); + } + + public static PendingObservation pendingObservation(AppendOverwriteMiddlewareContext context, + BrAPIObservationDAO brAPIObservationDAO, + OntologyService ontologyService, + ExperimentUtilities experimentUtilities) { + return new PendingObservation(context, brAPIObservationDAO, ontologyService, experimentUtilities); + } + + public static PendingLocation pendingLocation(AppendOverwriteMiddlewareContext context, + ProgramLocationService programLocationService, + LocationService locationService, + ExperimentUtilities experimentUtilities) { + return new PendingLocation(context, programLocationService, locationService, experimentUtilities); + } + + @Bean + @Prototype + public PendingTrial pendingTrialBean(AppendOverwriteMiddlewareContext context) { + return pendingTrial(context, trialService, brapiTrialDAO, experimentUtilities); + } + + @Bean + @Prototype + public PendingObservationUnit pendingObservationUnitBean(AppendOverwriteMiddlewareContext context) { + return pendingObservationUnit(context, observationUnitDAO, observationUnitService, experimentUtilities); + } + + @Bean + @Prototype + public PendingStudy pendingStudyBean(AppendOverwriteMiddlewareContext context) { + return pendingStudy(context, studyService, brAPIStudyDAO, experimentUtilities); + } + + @Bean + @Prototype + public PendingGermplasm pendingGermplasmBean(AppendOverwriteMiddlewareContext context) { + return pendingGermplasm(context, germplasmService, experimentUtilities); + } + + @Bean + @Prototype + public PendingDataset pendingDatasetBean(AppendOverwriteMiddlewareContext context) { + return pendingDataset(context, brAPIListDAO, datasetService, experimentUtilities); + } + + @Bean + @Prototype + public PendingObservation pendingObservationBean(AppendOverwriteMiddlewareContext context) { + return pendingObservation(context, brAPIObservationDAO, ontologyService, experimentUtilities); + } + + @Bean + @Prototype + public PendingLocation pendingLocationBean(AppendOverwriteMiddlewareContext context) { + return pendingLocation(context, programLocationService, locationService, experimentUtilities); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java new file mode 100644 index 000000000..0076e686e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingGermplasm.java @@ -0,0 +1,171 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; + +import io.micronaut.context.annotation.Prototype; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.GermplasmService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import java.util.*; +import java.util.stream.Collectors; + +@Prototype +public class PendingGermplasm implements ExperimentImportEntity { + AppendOverwriteWorkflowContext cache; + ImportContext importContext; + GermplasmService germplasmService; + ExperimentUtilities experimentUtilities; + + public PendingGermplasm(AppendOverwriteMiddlewareContext context, + GermplasmService germplasmService, + ExperimentUtilities experimentUtilities) { + this.cache = context.getAppendOverwriteWorkflowContext(); + this.importContext = context.getImportContext(); + this.germplasmService = germplasmService; + this.experimentUtilities = experimentUtilities; + } + /** + * Create new objects generated by the workflow in the BrAPI service. + * + * @param members List of entities to be created + * @return List of created entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiPost(List members) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + return null; + } + + /** + * Fetch objects required by the workflow from the BrAPI service. + * + * @return List of fetched entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiRead() throws ApiException { + // Get the dbIds of the germplasm belonging to the required exp units + Set germplasmDbIds = cache.getObservationUnitByNameNoScope().values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + + // Get the dataset belonging to required exp units + return germplasmService.fetchGermplasmByDbId(new HashSet<>(germplasmDbIds), importContext.getProgram()); + } + + /** + * Commit objects changed by the workflow to the BrAPI service. + * + * @param members List of entities to be updated + * @return List of updated entities + * @throws ApiException if there is an issue with the API call + * @throws IllegalArgumentException if method arguments are invalid + */ + @Override + public List brapiPut(List members) throws ApiException, IllegalArgumentException { + return new ArrayList<>(); + } + + /** + * Remove objects created by the workflow from the BrAPI service. + * + * @param members List of entities to be deleted + * @return true if deletion is successful, false otherwise + * @throws ApiException if there is an issue with the API call + */ + @Override + public boolean brapiDelete(List members) throws ApiException { + return false; + } + + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the BrAPI service. + * + * @param status State of the objects + * @return List of deep copies of entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List getBrAPIState(ImportObjectState status) throws ApiException { + return new ArrayList<>(); + } + + /** + * For workflow pending import objects of a given state, construct deep copies of the objects from the workflow context. + * + * @param status State of the objects + * @return List of deep copies of entities from workflow context + */ + @Override + public List copyWorkflowMembers(ImportObjectState status) { + return experimentUtilities.copyWorkflowCachePendingBrAPIObjects(cache.getExistingGermplasmByGID(), BrAPIGermplasm.class, status); + } + + /** + * For objects in the workflow context, update any foreign-key fields with values generated by the BrAPI service. + * + * @param members List of entities to be updated + */ + @Override + public void updateWorkflow(List members) { + + } + + /** + * Populate the workflow context with objects needed by the workflow. + * + * @param members List of entities to be initialized + */ + @Override + public void initializeWorkflow(List members) { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIGermplasm.class)) { + return; + } + + // Construct the pending germplasm from the BrAPI locations + List> pendingGermplasm = members.stream().map(g -> (BrAPIGermplasm) g).map(germplasmService::constructPIOFromBrapiGermplasm).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending germplasm by gid + Map> pendingGermplasmByGID = pendingGermplasm.stream().collect(Collectors.toMap(germplasmService::getGIDFromGermplasmPIO, pio -> pio)); + + // Construct a hashmap to look up the pending germplasm by the observation unit ID of a unit stored in the BrAPI service + Map> pendingGermplasmByOUId = cache.getPendingObsUnitByOUId().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> { + String gid = Optional.ofNullable(e.getValue().getBrAPIObject().getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.GID).getAsString()) + .orElseThrow(() -> new IllegalStateException("GID not set for unit: " + e.getKey())); + return Optional.ofNullable(pendingGermplasmByGID.get(gid)).orElseThrow(() -> new IllegalStateException("Observation unit missing germplasm: " + e.getKey())); + } + )); + + // Add the maps to the context for use in processing import + cache.setExistingGermplasmByGID(pendingGermplasmByGID); + cache.setPendingGermplasmByOUId(pendingGermplasmByOUId); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java new file mode 100644 index 000000000..516873f22 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingLocation.java @@ -0,0 +1,209 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; + +import io.micronaut.context.annotation.Prototype; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.ProgramLocationService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import java.util.*; +import java.util.stream.Collectors; + +@Prototype +public class PendingLocation implements ExperimentImportEntity { + AppendOverwriteWorkflowContext cache; + ImportContext importContext; + ProgramLocationService programLocationService; + LocationService locationService; + ExperimentUtilities experimentUtilities; + + public PendingLocation(AppendOverwriteMiddlewareContext context, + ProgramLocationService programLocationService, + LocationService locationService, + ExperimentUtilities experimentUtilities) { + this.cache = context.getAppendOverwriteWorkflowContext(); + this.importContext = context.getImportContext(); + this.programLocationService = programLocationService; + this.locationService = locationService; + this.experimentUtilities = experimentUtilities; + } + + /** + * Create new objects generated by the workflow in the BrAPI service. + * + * @param members List of entities to be created + * @return List of created entities + */ + @Override + public List brapiPost(List members) throws MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + // Construct requests + List locationRequests = members.stream() + .map(location -> ProgramLocationRequest.builder() + .name(location.getName()) + .build()) + .collect(Collectors.toList()); + + // Create acting user + AuthenticatedUser actingUser = new AuthenticatedUser(importContext.getUpload().getUpdatedByUser().getName(), new ArrayList<>(), importContext.getUpload().getUpdatedByUser().getId(), new ArrayList<>()); + + return programLocationService.create(actingUser, importContext.getProgram().getId(), locationRequests); + } + + /** + * Fetch objects required by the workflow from the BrAPI service. + * + * @return List of fetched entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiRead() throws ApiException { + // Get the dbIds of the studies belonging to the required exp units + Set locationDbIds = cache.getStudyByNameNoScope().values().stream().map(pio -> pio.getBrAPIObject().getLocationDbId()).collect(Collectors.toSet()); + + // Get the locations belonging to required exp units + return locationService.fetchLocationsByDbId(locationDbIds, importContext.getProgram()); + } + + /** + * Commit objects changed by the workflow to the BrAPI service. + * + * @param members List of entities to be updated + * @return List of updated entities + * @throws IllegalArgumentException if method arguments are invalid + */ + @Override + public List brapiPut(List members) throws IllegalArgumentException { + return new ArrayList<>(); + } + + /** + * Remove objects created by the workflow from the BrAPI service. + * + * @param members List of entities to be deleted + * @return true if deletion is successful, false otherwise + */ + @Override + public boolean brapiDelete(List members) { + // TODO: implement delete for program locations + return false; + } + + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the BrAPI service. + * + * @param status State of the objects + * @return List of deep copies of entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List getBrAPIState(ImportObjectState status) throws ApiException { + return new ArrayList<>(); + } + + /** + * For workflow pending import objects of a given state, construct deep copies of the objects from the workflow context. + * + * @param status State of the objects + * @return List of deep copies of entities from workflow context + */ + @Override + public List copyWorkflowMembers(ImportObjectState status) { + return experimentUtilities.copyWorkflowCachePendingBrAPIObjects(cache.getLocationByName(), ProgramLocation.class, status); + } + + /** + * For objects in the workflow context, update any foreign-key fields with values generated by the BrAPI service. + * + * @param members List of entities to be updated + */ + @Override + public void updateWorkflow(List members) { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, ProgramLocation.class)) { + return; + } + + for (U member : members) { + ProgramLocation location = (ProgramLocation) member; + + // Set the system-generated dbId for each newly created location + cache.getLocationByName().get(location.getName()).getBrAPIObject().setLocationDbId(location.getLocationDbId()); + + // Set the location dbid for cached studies + cache.getStudyByNameNoScope().values().stream() + .filter(study -> location.getId().toString().equals(study.getBrAPIObject().getLocationDbId())) + .forEach(study -> study.getBrAPIObject().setLocationDbId(location.getLocationDbId())); + } + + } + + /** + * Populate the workflow context with objects needed by the workflow. + * + * @param members List of entities to be initialized + */ + @Override + public void initializeWorkflow(List members) { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, ProgramLocation.class)) { + return; + } + + // Construct the pending locations from the BrAPI locations + List> pendingLocations = members.stream().map((U brapiLocation) -> locationService.constructPIOFromBrapiLocation((ProgramLocation) brapiLocation)).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending location by location name + Map> pendingLocationByName = pendingLocations.stream().collect(Collectors.toMap(pio -> pio.getBrAPIObject().getName(), pio -> pio)); + + // Construct a hashmap to look up the pending location by the observation unit ID of a unit stored in the BrAPI service + Map> pendingLocationByOUId = cache.getPendingObsUnitByOUId().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> { + String name = Optional.ofNullable(e.getValue().getBrAPIObject().getLocationName()) + .orElseGet(() -> { + PendingImportObject pendingStudy = cache.getPendingStudyByOUId().get(e.getKey()); + if (pendingStudy == null) { + throw new IllegalStateException("Observation unit missing study: " + e.getKey()); + } + return pendingStudy.getBrAPIObject().getLocationName(); + }); + return Optional.ofNullable(pendingLocationByName.get(name)) + .orElseThrow(() -> new IllegalStateException("Observation unit missing location: " + e.getKey())); + } + )); + + // Add the maps to the context for use in processing import + cache.setLocationByName(pendingLocationByName); + cache.setPendingLocationByOUId(pendingLocationByOUId); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java new file mode 100644 index 000000000..edf63408c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservation.java @@ -0,0 +1,197 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; + +import io.micronaut.context.annotation.Prototype; +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.DoesNotExistException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +@Prototype +public class PendingObservation implements ExperimentImportEntity { + AppendOverwriteWorkflowContext cache; + ImportContext importContext; + BrAPIObservationDAO brAPIObservationDAO; + OntologyService ontologyService; + ExperimentUtilities experimentUtilities; + + public PendingObservation(AppendOverwriteMiddlewareContext context, + BrAPIObservationDAO brAPIObservationDAO, + OntologyService ontologyService, + ExperimentUtilities experimentUtilities) { + this.cache = context.getAppendOverwriteWorkflowContext(); + this.importContext = context.getImportContext(); + this.brAPIObservationDAO = brAPIObservationDAO; + this.ontologyService = ontologyService; + this.experimentUtilities = experimentUtilities; + } + + + /** + * Create new objects generated by the workflow in the BrAPI service. + * + * @param members List of entities to be created + * @return List of created entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiPost(List members) throws ApiException, DoesNotExistException { + // TODO: move the trait setting out to a higher level + // Fetch the program traits + List traits = ontologyService.getTraitsByProgramId(importContext.getProgram().getId(), true); + CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); + for ( Trait trait: traits) { + traitMap.put(trait.getObservationVariableName(),trait); + } + + // Set the trait dbId on the observation requests + for (BrAPIObservation observation : members) { + String observationVariableName = observation.getObservationVariableName(); + if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { + String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); + observation.setObservationVariableDbId(observationVariableDbId); + } + } + + // TODO: move this logic out to a higher level + // Do not create observations in the BrAPI service if there is no value + List nonBlankMembers = members.stream().filter(obs -> !obs.getValue().isBlank()).collect(Collectors.toList()); + + // Create the observations + return brAPIObservationDAO.createBrAPIObservations(nonBlankMembers, importContext.getProgram().getId(), importContext.getUpload()); + } + + /** + * Fetch objects required by the workflow from the BrAPI service. + * + * @return List of fetched entities + */ + @Override + public List brapiRead() { + return new ArrayList<>(); + } + + /** + * Commit objects changed by the workflow to the BrAPI service. + * + * @param members List of entities to be updated + * @return List of updated entities + * @throws ApiException if there is an issue with the API call + * @throws IllegalArgumentException if method arguments are invalid + */ + @Override + public List brapiPut(List members) throws ApiException, IllegalArgumentException { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIObservation.class)) { + return new ArrayList(); + } + + List updatedObservations = new ArrayList<>(); + for (U member : members) { + BrAPIObservation observation = (BrAPIObservation) member; + Optional.ofNullable(brAPIObservationDAO.updateBrAPIObservation(observation.getObservationDbId(), observation, importContext.getProgram().getId())).ifPresent(updatedObservations::add); + } + + return (List) updatedObservations; + } + + /** + * Remove objects created by the workflow from the BrAPI service. + * + * @param members List of entities to be deleted + * @return true if deletion is successful, false otherwise + * @throws ApiException if there is an issue with the API call + */ + @Override + public boolean brapiDelete(List members) throws ApiException { + // TODO: implement delete for observations on BrapiJavaTestServer + return false; + } + + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the BrAPI service. + * + * @param status State of the objects + * @return List of deep copies of entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List getBrAPIState(ImportObjectState status) throws ApiException { + List ids = copyWorkflowMembers(status).stream().map(BrAPIObservation::getObservationDbId).collect(Collectors.toList()); + return brAPIObservationDAO.getObservationsByDbIds(ids, importContext.getProgram()); + } + + /** + * For workflow pending import objects of a given state, construct deep copies of the objects from the workflow context. + * + * @param status State of the objects + * @return List of deep copies of entities from workflow context + */ + @Override + public List copyWorkflowMembers(ImportObjectState status) { + return experimentUtilities.copyWorkflowCachePendingBrAPIObjects(cache.getPendingObservationByHash(), BrAPIObservation.class, status); + } + + /** + * For objects in the workflow context, update any foreign-key fields with values generated by the BrAPI service. + * + * @param members List of entities to be updated + */ + @Override + public void updateWorkflow(List members) { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIObservation.class)) { + return; + } + + // Update the workflow ref by setting the system-generated dbId for each newly created trial + for (U member : members) { + BrAPIObservation observation = (BrAPIObservation) member; + // TODO:set the observation dBId + } + } + + /** + * Populate the workflow context with objects needed by the workflow. + * + * @param members List of entities to be initialized + */ + @Override + public void initializeWorkflow(List members) { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIObservation.class)) { + return; + } + + // TODO:add previous observations + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java new file mode 100644 index 000000000..6bb682ee2 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java @@ -0,0 +1,225 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; + +import io.micronaut.context.annotation.Prototype; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; + +import java.util.*; +import java.util.stream.Collectors; + +@Prototype +public class PendingObservationUnit implements ExperimentImportEntity { + AppendOverwriteWorkflowContext cache; + ImportContext importContext; + BrAPIObservationUnitDAO observationUnitDAO; + ObservationUnitService observationUnitService; + ExperimentUtilities experimentUtilities; + + public PendingObservationUnit(AppendOverwriteMiddlewareContext context, + BrAPIObservationUnitDAO observationUnitDAO, + ObservationUnitService observationUnitService, + ExperimentUtilities experimentUtilities) { + this.cache = context.getAppendOverwriteWorkflowContext(); + this.importContext = context.getImportContext(); + this.observationUnitDAO = observationUnitDAO; + this.observationUnitService = observationUnitService; + this.experimentUtilities = experimentUtilities; + } + /** + * Create new objects generated by the workflow in the BrAPI service. + * + * @param members List of entities to be created + * @return List of created entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiPost(List members) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + // TODO: move the germplasm foreign key assignment out one level + // Set the germplasm foreign key for observation unit requests + cache.getExistingGermplasmByGID().values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(germplasm -> { + + // Collect all observation units that are associated with a germplasm GID + members.stream().filter(obsUnit -> germplasm.getAccessionNumber() != null && + germplasm.getAccessionNumber().equals(obsUnit + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.GID).getAsString())) + + // Set the foreign key for each unit + .forEach(obsUnit -> obsUnit.setGermplasmDbId(germplasm.getGermplasmDbId())); + }); + + return observationUnitDAO.createBrAPIObservationUnits(members, importContext.getProgram().getId(), importContext.getUpload()); + } + + /** + * Fetch objects required by the workflow from the BrAPI service. + * + * @return List of fetched entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiRead() throws ApiException { + // Collect deltabreed-generated exp unit ids listed in the import + Set expUnitIds = cache.getReferenceOUIds(); + + // For each id fetch the observation unit from the brapi data store + return observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), importContext.getProgram()); + } + + /** + * Commit objects changed by the workflow to the BrAPI service. + * + * @param members List of entities to be updated + * @return List of updated entities + * @throws ApiException if there is an issue with the API call + * @throws IllegalArgumentException if method arguments are invalid + */ + @Override + public List brapiPut(List members) throws ApiException, IllegalArgumentException { + return null; + } + + /** + * Remove objects created by the workflow from the BrAPI service. + * + * @param members List of entities to be deleted + * @return true if deletion is successful, false otherwise + * @throws ApiException if there is an issue with the API call + */ + @Override + public boolean brapiDelete(List members) throws ApiException { + return false; + } + + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the BrAPI service. + * + * @param status State of the objects + * @return List of deep copies of entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List getBrAPIState(ImportObjectState status) throws ApiException { + return null; + } + + /** + * For workflow pending import objects of a given state, construct deep copies of the objects from the workflow context. + * + * @param status State of the objects + * @return List of deep copies of entities from workflow context + */ + @Override + public List copyWorkflowMembers(ImportObjectState status) { + return experimentUtilities.copyWorkflowCachePendingBrAPIObjects(cache.getObservationUnitByNameNoScope(), BrAPIObservationUnit.class, status); + } + + /** + * For objects in the workflow context, update any foreign-key fields with values generated by the BrAPI service. + * + * @param members List of entities to be updated + */ + @Override + public void updateWorkflow(List members) { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIObservationUnit.class)) { + return; + } + + for (U member : members) { + BrAPIObservationUnit unit = (BrAPIObservationUnit) member; + + // Set the dbId for observation units + String studyNameNoScope = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), importContext.getProgram().getKey()); + String unitNameNoScope = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getObservationUnitName(), importContext.getProgram().getKey()); + String key = studyNameNoScope + unitNameNoScope; + cache.getObservationUnitByNameNoScope().get(key).getBrAPIObject().setObservationUnitDbId(unit.getObservationUnitDbId()); + + // Set the unit dbId for observations connected with the unit, matching on environment and exp unit + cache.getPendingObservationByHash().values() + .stream() + .filter(obs -> obs.getBrAPIObject() + .getAdditionalInfo() != null + && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null + && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) + .getAsString() + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), importContext.getProgram().getKey())) + && Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getBrAPIObject().getObservationUnitName(), importContext.getProgram().getKey()) + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getObservationUnitName(), importContext.getProgram().getKey())) + ) + .forEach(obs -> { + if (StringUtils.isBlank(obs.getBrAPIObject().getObservationUnitDbId())) { + obs.getBrAPIObject().setObservationUnitDbId(unit.getObservationUnitDbId()); + } + obs.getBrAPIObject().setStudyDbId(unit.getStudyDbId()); + obs.getBrAPIObject().setGermplasmDbId(unit.getGermplasmDbId()); + }); + } + } + + /** + * Populate the workflow context with objects needed by the workflow. + * + * @param members List of entities to be initialized + */ + @Override + public void initializeWorkflow(List members) { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIObservationUnit.class)) { + return; + } + + // Construct pending import objects from the units + List> pendingUnits = members.stream().map(u -> (BrAPIObservationUnit) u).map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending unit by ID + Map> pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); + + // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed + Map> pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), importContext.getProgram()); + + // add maps to the context for use in processing import + cache.setPendingObsUnitByOUId(pendingUnitById); + cache.setObservationUnitByNameNoScope(pendingUnitByNameNoScope); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java new file mode 100644 index 000000000..546ffff73 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingStudy.java @@ -0,0 +1,196 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; + +import io.micronaut.context.annotation.Prototype; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; + +import java.util.*; +import java.util.stream.Collectors; + +@Prototype +public class PendingStudy implements ExperimentImportEntity{ + AppendOverwriteWorkflowContext cache; + ImportContext importContext; + StudyService studyService; + BrAPIStudyDAO brAPIStudyDAO; + ExperimentUtilities experimentUtilities; + + public PendingStudy(AppendOverwriteMiddlewareContext context, + StudyService studyService, + BrAPIStudyDAO brAPIStudyDAO, + ExperimentUtilities experimentUtilities) { + this.cache = context.getAppendOverwriteWorkflowContext(); + this.importContext = context.getImportContext(); + this.studyService = studyService; + this.brAPIStudyDAO = brAPIStudyDAO; + this.experimentUtilities = experimentUtilities; + } + /** + * Create new objects generated by the workflow in the BrAPI service. + * + * @param members List of entities to be created + * @return List of created entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiPost(List members) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + return brAPIStudyDAO.createBrAPIStudies(members, importContext.getProgram().getId(), importContext.getUpload()); + } + + /** + * Fetch objects required by the workflow from the BrAPI service. + * + * @return List of fetched entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiRead() throws ApiException { + // Get the dbIds of the studies belonging to the required exp units + Set studyDbIds = cache.getObservationUnitByNameNoScope().values().stream().map(studyService::getStudyDbIdBelongingToPendingUnit).collect(Collectors.toSet()); + return studyService.fetchBrapiStudiesByDbId(studyDbIds, importContext.getProgram()); + } + + /** + * Commit objects changed by the workflow to the BrAPI service. + * + * @param members List of entities to be updated + * @return List of updated entities + * @throws ApiException if there is an issue with the API call + * @throws IllegalArgumentException if method arguments are invalid + */ + @Override + public List brapiPut(List members) throws ApiException, IllegalArgumentException { + return new ArrayList<>(); + } + + /** + * Remove objects created by the workflow from the BrAPI service. + * + * @param members List of entities to be deleted + * @return true if deletion is successful, false otherwise + * @throws ApiException if there is an issue with the API call + */ + @Override + public boolean brapiDelete(List members) throws ApiException { + // TODO: implement delete study endpoint on BrAPIJavaTestServer + return false; + } + + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the BrAPI service. + * + * @param status State of the objects + * @return List of deep copies of entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List getBrAPIState(ImportObjectState status) throws ApiException { + return new ArrayList<>(); + } + + /** + * For workflow pending import objects of a given state, construct deep copies of the objects from the workflow context. + * + * @param status State of the objects + * @return List of deep copies of entities from workflow context + */ + @Override + public List copyWorkflowMembers(ImportObjectState status) { + return experimentUtilities.copyWorkflowCachePendingBrAPIObjects(cache.getStudyByNameNoScope(), BrAPIStudy.class, status); + } + + /** + * For objects in the workflow context, update any foreign-key fields with values generated by the BrAPI service. + * + * @param members List of entities to be updated + */ + @Override + public void updateWorkflow(List members) { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIStudy.class)) { + return; + } + + for (U member : members) { + BrAPIStudy study = (BrAPIStudy) member; + + // set the DbId for each newly created study + String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), importContext.getProgram().getKey()); + cache.getStudyByNameNoScope().get(createdStudy_name_no_key).getBrAPIObject().setStudyDbId(study.getStudyDbId()); + + // Set the study dbId for observation units + cache.getObservationUnitByNameNoScope().values() + .stream() + .filter(obsUnit -> obsUnit.getBrAPIObject() + .getStudyName() + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), importContext.getProgram().getKey()))) + .forEach(obsUnit -> { + obsUnit.getBrAPIObject().setStudyDbId(study.getStudyDbId()); + obsUnit.getBrAPIObject().setTrialDbId(study.getTrialDbId()); + }); + } + } + + /** + * Populate the workflow context with objects needed by the workflow. + * + * @param members List of entities to be initialized + */ + @Override + public void initializeWorkflow(List members) { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPIStudy.class)) { + return; + } + + // Construct the pending studies from the BrAPI studies + List> pendingStudies = members.stream() + .map(s->(BrAPIStudy) s) + .map(pio -> studyService.constructPIOFromBrapiStudy(pio, importContext.getProgram())).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending study by study name with the program key removed + Map> pendingStudyByNameNoScope = pendingStudies.stream() + .collect(Collectors.toMap(pio -> Utilities.removeProgramKeyAndUnknownAdditionalData(pio.getBrAPIObject().getStudyName(), importContext.getProgram().getKey()), pio -> pio)); + + // Construct a hashmap to look up the pending study by the observation unit ID of a unit stored in the BrAPI service + Map> pendingStudyByOUId = cache.getPendingObsUnitByOUId().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, + e -> Optional.ofNullable(e.getValue().getBrAPIObject().getStudyName()) + .map(studyNameScoped -> Utilities.removeProgramKeyAndUnknownAdditionalData(studyNameScoped, importContext.getProgram().getKey())) + .map(pendingStudyByNameNoScope::get) + .orElseThrow(() -> new IllegalStateException("Observation unit missing study name: " + e.getKey())))); + + // Add the maps to the context for use in processing import + cache.setStudyByNameNoScope(pendingStudyByNameNoScope); + cache.setPendingStudyByOUId(pendingStudyByOUId); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java new file mode 100644 index 000000000..9c0c80d34 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingTrial.java @@ -0,0 +1,228 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; + +import io.micronaut.context.annotation.Prototype; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; +import org.breedinginsight.utilities.Utilities; + +import java.util.*; +import java.util.stream.Collectors; + +@Prototype +public class PendingTrial implements ExperimentImportEntity { + private final AppendOverwriteWorkflowContext cache; + private final ImportContext importContext; + private final TrialService trialService; + private final BrAPITrialDAO brapiTrialDAO; + private final ExperimentUtilities experimentUtilities; + + public PendingTrial(AppendOverwriteMiddlewareContext context, + TrialService trialService, + BrAPITrialDAO brapiTrialDAO, + ExperimentUtilities experimentUtilities) { + this.cache = context.getAppendOverwriteWorkflowContext(); + this.importContext = context.getImportContext(); + this.trialService = trialService; + this.brapiTrialDAO = brapiTrialDAO; + this.experimentUtilities = experimentUtilities; + } + + /** + * Create new objects generated by the workflow in the BrAPI service. + * + * @param members List of entities to be created + * @return List of created entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiPost(List members) throws ApiException { + return brapiTrialDAO.createBrAPITrials(members, importContext.getProgram().getId(), importContext.getUpload()); + } + + /** + * Fetch objects required by the workflow from the BrAPI service. + * + * @return List of fetched entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List brapiRead() throws ApiException { + // Get the dbIds of the trials belonging to the required exp units + Set trialDbIds = Optional.ofNullable(cache.getObservationUnitByNameNoScope()).map(Map::values) + .orElse(Collections.emptySet()) + .stream() + .map(pendingUnit -> trialService.getTrialDbIdBelongingToPendingUnit(pendingUnit, importContext.getProgram())) + .collect(Collectors.toSet()); + + // Get the BrAPI trials belonging to required exp units + return trialService.fetchBrapiTrialsByDbId(trialDbIds, importContext.getProgram()); + } + + /** + * Commit objects changed by the workflow to the BrAPI service. + * + * @param members List of entities to be updated + * @return List of updated entities + * @throws ApiException if there is an issue with the API call + * @throws IllegalArgumentException if method arguments are invalid + */ + @Override + public List brapiPut(List members) throws ApiException, IllegalArgumentException { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPITrial.class)) { + return new ArrayList(); + } + + List updatedTrials = new ArrayList<>(); + for (U member : members) { + BrAPITrial trial = (BrAPITrial) member; + Optional.ofNullable(brapiTrialDAO.updateBrAPITrial(trial.getTrialDbId(), trial, importContext.getProgram().getId())).ifPresent(updatedTrials::add); + } + + return (List) updatedTrials; + } + + /** + * Remove objects created by the workflow from the BrAPI service. + * + * @param members List of entities to be deleted + * @return true if deletion is successful, false otherwise + * @throws ApiException if there is an issue with the API call + */ + @Override + public boolean brapiDelete(List members) throws ApiException { + // TODO: implement delete for trials on BrapiJavaTestServer + return false; + } + + /** + * For workflow pending import objects of a given state, fetch deep copies of the objects from the BrAPI service. + * + * @param status State of the objects + * @return List of deep copies of entities + * @throws ApiException if there is an issue with the API call + */ + @Override + public List getBrAPIState(ImportObjectState status) throws ApiException { + List ids = copyWorkflowMembers(status).stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList()); + return brapiTrialDAO.getTrialsByDbIds(ids, importContext.getProgram()); + } + + /** + * For workflow pending import objects of a given state, construct deep copies of the objects from the workflow context. + * + * @param status State of the objects + * @return List of deep copies of entities from workflow context + */ + @Override + public List copyWorkflowMembers(ImportObjectState status) { + return experimentUtilities.copyWorkflowCachePendingBrAPIObjects(cache.getTrialByNameNoScope(), BrAPITrial.class, status); + } + + /** + * For objects in the workflow context, update any foreign-key fields with values generated by the BrAPI service. + * + * @param members List of entities to be updated + */ + @Override + public void updateWorkflow(List members) { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPITrial.class)) { + return; + } + + // Update the workflow ref by setting the system-generated dbId for each newly created trial + for (U member : members) { + BrAPITrial trial = (BrAPITrial) member; + String createdTrialNameNoScope = Utilities.removeProgramKey(trial.getTrialName(), importContext.getProgram().getKey()); + cache.getTrialByNameNoScope().get(createdTrialNameNoScope).getBrAPIObject().setTrialDbId(trial.getTrialDbId()); + } + + // Update trial DbIds in studies for all distinct trials + cache.getTrialByNameNoScope().values().stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(trial -> + cache.getStudyByNameNoScope().values().stream() + .filter(study -> study.getBrAPIObject().getTrialName() + .equals(Utilities.removeProgramKey(trial.getTrialName(), importContext.getProgram().getKey()))) + .forEach(study -> study.getBrAPIObject().setTrialDbId(trial.getTrialDbId())) + ); + } + + /** + * Populate the workflow context with objects needed by the workflow. + * + * @param members List of entities to be initialized + */ + @Override + public void initializeWorkflow(List members) { + // Check if the input list is of type List + if (experimentUtilities.isInvalidMemberListForClass(members, BrAPITrial.class)) { + return; + } + + // Construct the pending trials from the BrAPI trials + List> pendingTrials = members.stream() + .map(t -> (BrAPITrial) t).map(trialService::constructPIOFromBrapiTrial).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending trial by trial name with the program key removed + Map> pendingTrialByNameNoScope = pendingTrials.stream() + .collect(Collectors.toMap(pio -> Utilities.removeProgramKey(pio.getBrAPIObject().getTrialName(), importContext.getProgram().getKey()), pio -> pio)); + + // Construct a hashmap to look up the pending trial by the observation unit ID of a unit stored in the BrAPI service + Map> pendingTrialByOUId = cache.getPendingObsUnitByOUId().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> { + String trialName = e.getValue().getBrAPIObject().getTrialName(); + String studyName = e.getValue().getBrAPIObject().getStudyName(); + + if (trialName != null) { + String nameNoScope = Utilities.removeProgramKeyAndUnknownAdditionalData(trialName, importContext.getProgram().getKey()); + return Optional.ofNullable(pendingTrialByNameNoScope.get(nameNoScope)) + .orElseThrow(() -> new IllegalStateException("Failed to find pending trial for observation unit" + e.getKey())); + } else if (studyName != null) { + String nameNoScope = Utilities.removeProgramKeyAndUnknownAdditionalData( + cache.getStudyByNameNoScope().get(studyName).getBrAPIObject().getTrialName(), + importContext.getProgram().getKey() + ); + return Optional.ofNullable(pendingTrialByNameNoScope.get(nameNoScope)) + .orElseThrow(() -> new IllegalStateException("Failed to find pending trial for observation unit" + e.getKey())); + } else { + throw new IllegalStateException("Observation Unit missing trial name and study name: " + e.getKey()); + } + } + )); + + // Add the maps to the context for use in processing import + cache.setTrialByNameNoScope(pendingTrialByNameNoScope); + cache.setPendingTrialByOUId(pendingTrialByOUId); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java new file mode 100644 index 000000000..14bad6915 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java @@ -0,0 +1,46 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; + +@Slf4j +@Prototype +public class AppendOverwriteIDValidation extends AppendOverwriteMiddleware { + @Override + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { + + context.getAppendOverwriteWorkflowContext().setReferenceOUIds(ExperimentUtilities.collateReferenceOUIds(context)); + return processNext(context); + } + + @Override + public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { + // tag an error if it occurred in this local transaction + context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); + + // undo the prior local transaction + return compensatePrior(context); + } + + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java new file mode 100644 index 000000000..1b6cbd7ee --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPICommit.java @@ -0,0 +1,56 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; + +import javax.inject.Inject; + +@Slf4j +@Prototype +public class BrAPICommit extends AppendOverwriteMiddleware { + AppendOverwriteMiddleware middleware; + @Inject + public BrAPICommit(BrAPIDatasetCommit brAPIDatasetCommit, + BrAPITrialCommit brAPITrialCommit, + LocationCommit locationCommit, + BrAPIStudyCommit brAPIStudyCommit, + BrAPIObservationUnitCommit brAPIObservationUnitCommit, + BrAPIObservationCommit brAPIObservationCommit) { + + // Note: the order is important because system-generated dbIds from prior steps are used as foreign keys in + // subsequent steps + this.middleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link( + brAPIDatasetCommit, + brAPITrialCommit, + locationCommit, + brAPIStudyCommit, + brAPIObservationUnitCommit, + brAPIObservationCommit); + } + + @Override + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { + log.debug("starting post of experiment data to BrAPI server"); + + return this.middleware.process(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java new file mode 100644 index 000000000..368673f19 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIDatasetCommit.java @@ -0,0 +1,92 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory.WorkflowUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.Optional; + +@Slf4j +@Prototype +public class BrAPIDatasetCommit extends AppendOverwriteMiddleware { + private final BrAPICreationFactory brAPICreationFactory; + private final BrAPIUpdateFactory brAPIUpdateFactory; + private Optional.BrAPICreationState> createdDatasets; + private Optional.BrAPIUpdateState> priorDatasets; + + @Inject + public BrAPIDatasetCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUpdateFactory brAPIUpdateFactory) { + this.brAPICreationFactory = brAPICreationFactory; + this.brAPIUpdateFactory = brAPIUpdateFactory; + } + @Override + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { + + try { + WorkflowCreation datasetCreation = brAPICreationFactory.datasetWorkflowCreationBean(context); + + log.info("creating new datasets in the BrAPI service"); + createdDatasets = datasetCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); + WorkflowUpdate datasetUpdate = brAPIUpdateFactory.datasetWorkflowUpdateBean(context); + priorDatasets = datasetUpdate.getBrAPIState(); + + log.info("adding new observation variables to datasets"); + datasetUpdate.execute().map(d -> (WorkflowUpdate.BrAPIUpdateState) d); + } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); + return this.compensate(context); + } + return processNext(context); + } + + @Override + public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { + // Tag an error if it occurred in this local transaction + context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); + + // Delete any created datasets + createdDatasets.ifPresent(WorkflowCreation.BrAPICreationState::undo); + + // Revert any changes made to datasets in the BrAPI service + priorDatasets.ifPresent(state -> { + try { + state.restore(); + } catch (ApiException e) { + log.error("Error trying to restore BrAPI variable state: " + Utilities.generateApiExceptionLogMessage(e), e); + } + }); + + // Undo the prior local transaction + return compensatePrior(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java new file mode 100644 index 000000000..779d75493 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationCommit.java @@ -0,0 +1,95 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory.WorkflowUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.Optional; + +@Slf4j +@Prototype +public class BrAPIObservationCommit extends AppendOverwriteMiddleware { + private final BrAPICreationFactory brAPICreationFactory; + private final BrAPIUpdateFactory brAPIUpdateFactory; + private Optional.BrAPICreationState> createdBrAPIObservations; + private Optional.BrAPIUpdateState> priorBrAPIObservations; + + @Inject + public BrAPIObservationCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUpdateFactory brAPIUpdateFactory) { + this.brAPICreationFactory = brAPICreationFactory; + this.brAPIUpdateFactory = brAPIUpdateFactory; + } + @Override + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { + try { + WorkflowCreation brAPIObservationCreation = brAPICreationFactory.observationWorkflowCreationBean(context); + + log.info("creating new observations in the BrAPI service"); + createdBrAPIObservations = brAPIObservationCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); + WorkflowUpdate brAPIObservationUpdate = brAPIUpdateFactory.observationWorkflowUpdateBean(context); + priorBrAPIObservations = brAPIObservationUpdate.getBrAPIState(); + + log.info("updating existing observations in the BrAPI service"); + brAPIObservationUpdate.execute().map(s -> (WorkflowUpdate.BrAPIUpdateState) s); + + } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); + return this.compensate(context); + } + + return processNext(context); + } + + @Override + public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { + // Tag an error if it occurred in this local transaction + context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); + + + // Delete any created observations from the BrAPI service + createdBrAPIObservations.ifPresent(WorkflowCreation.BrAPICreationState::undo); + + // Revert any changes made to observations in the BrAPI service + priorBrAPIObservations.ifPresent(state -> { + try { + state.restore(); + } catch (ApiException e) { + log.error("Error trying to restore BrAPI variable state: " + Utilities.generateApiExceptionLogMessage(e), e); + } + }); + + // Undo the prior local transaction + return compensatePrior(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java new file mode 100644 index 000000000..4610be387 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIObservationUnitCommit.java @@ -0,0 +1,71 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import javax.inject.Inject; +import java.util.Optional; + +@Slf4j +@Prototype +public class BrAPIObservationUnitCommit extends AppendOverwriteMiddleware { + private final BrAPICreationFactory brAPICreationFactory; + private Optional.BrAPICreationState> createdBrAPIObservationUnits; + + @Inject + public BrAPIObservationUnitCommit(BrAPICreationFactory brAPICreationFactory) { + this.brAPICreationFactory = brAPICreationFactory; + } + @Override + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { + try{ + WorkflowCreation brAPIObservationUnitCreation = brAPICreationFactory.observationUnitWorkflowCreationBean(context); + + log.info("creating new observation units in the BrAPI service"); + createdBrAPIObservationUnits = brAPIObservationUnitCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); + } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); + return this.compensate(context); + } + return processNext(context); + } + + @Override + public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { + // Tag an error if it occurred in this local transaction + context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); + + // Delete any created trials from the BrAPI service + createdBrAPIObservationUnits.ifPresent(WorkflowCreation.BrAPICreationState::undo); + + // Undo the prior local transaction + return compensatePrior(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java new file mode 100644 index 000000000..f10a0b8b6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPIStudyCommit.java @@ -0,0 +1,72 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import javax.inject.Inject; +import java.util.Optional; + +@Slf4j +@Prototype +public class BrAPIStudyCommit extends AppendOverwriteMiddleware { + private final BrAPICreationFactory brAPICreationFactory; + private Optional.BrAPICreationState> createdBrAPIStudies; + + @Inject + public BrAPIStudyCommit(BrAPICreationFactory brAPICreationFactory) { + this.brAPICreationFactory = brAPICreationFactory; + } + @Override + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { + try { + WorkflowCreation brAPIStudyCreation = brAPICreationFactory.studyWorkflowCreationBean(context); + + log.info("creating new studies in the BrAPI service"); + createdBrAPIStudies = brAPIStudyCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); + } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); + return this.compensate(context); + } + + return processNext(context); + } + + @Override + public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { + // Tag an error if it occurred in this local transaction + context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); + + // Delete any created studies from the BrAPI service + createdBrAPIStudies.ifPresent(WorkflowCreation.BrAPICreationState::undo); + + // Undo the prior local transaction + return compensatePrior(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java new file mode 100644 index 000000000..3ca112b21 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/BrAPITrialCommit.java @@ -0,0 +1,93 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIUpdateFactory.WorkflowUpdate; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.Optional; + +@Slf4j +@Prototype +public class BrAPITrialCommit extends AppendOverwriteMiddleware { + private final BrAPICreationFactory brAPICreationFactory; + private final BrAPIUpdateFactory brAPIUpdateFactory; + private Optional.BrAPICreationState> createdBrAPITrials; + private Optional.BrAPIUpdateState> priorBrAPITrials; + + @Inject + public BrAPITrialCommit(BrAPICreationFactory brAPICreationFactory, BrAPIUpdateFactory brAPIUpdateFactory) { + this.brAPICreationFactory = brAPICreationFactory; + this.brAPIUpdateFactory = brAPIUpdateFactory; + } + @Override + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { + try { + WorkflowCreation brAPITrialCreation = brAPICreationFactory.trialWorkflowCreationBean(context); + + log.info("creating new trials in the BrAPI service"); + createdBrAPITrials = brAPITrialCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); + WorkflowUpdate brAPITrialUpdate = brAPIUpdateFactory.trialWorkflowUpdateBean(context); + priorBrAPITrials = brAPITrialUpdate.getBrAPIState(); + + log.info("updating existing trials in the BrAPI service"); + brAPITrialUpdate.execute().map(s -> (WorkflowUpdate.BrAPIUpdateState) s); + + } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); + return this.compensate(context); + } + + return processNext(context); + } + + @Override + public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { + // Tag an error if it occurred in this local transaction + context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); + + // Delete any created trials from the BrAPI service + createdBrAPITrials.ifPresent(WorkflowCreation.BrAPICreationState::undo); + + // Revert any changes made to trials in the BrAPI service + priorBrAPITrials.ifPresent(state -> { + try { + state.restore(); + } catch (ApiException e) { + log.error("Error trying to restore BrAPI variable state: " + Utilities.generateApiExceptionLogMessage(e), e); + } + }); + + // Undo the prior local transaction + return compensatePrior(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java new file mode 100644 index 000000000..34fda1fca --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/LocationCommit.java @@ -0,0 +1,71 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPICreationFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowCreation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; + +import javax.inject.Inject; +import java.util.Optional; + +@Slf4j +@Prototype +public class LocationCommit extends AppendOverwriteMiddleware { + private final BrAPICreationFactory brAPICreationFactory; + private Optional createdLocations; + + @Inject + public LocationCommit(BrAPICreationFactory brAPICreationFactory) { + this.brAPICreationFactory = brAPICreationFactory; + } + @Override + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { + try { + WorkflowCreation locationCreation = brAPICreationFactory.locationWorkflowCreationBean(context); + log.info("creating new locationss in the Deltabreed database"); + createdLocations = locationCreation.execute().map(s -> (WorkflowCreation.BrAPICreationState) s); + } catch (ApiException | MissingRequiredInfoException | UnprocessableEntityException | DoesNotExistException e) { + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); + return this.compensate(context); + } + return processNext(context); + } + + @Override + public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { + // Tag an error if it occurred in this local transaction + context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); + + // Delete any created locations + createdLocations.ifPresent(WorkflowCreation.BrAPICreationState::undo); + + // Undo the prior local transaction + return compensatePrior(context); + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java new file mode 100644 index 000000000..e65d11d15 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -0,0 +1,76 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIReadFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowReadInitialization; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.model.ProgramLocation; + +import javax.inject.Inject; + +@Slf4j +@Prototype +public class WorkflowInitialization extends AppendOverwriteMiddleware { + WorkflowReadInitialization brAPIObservationUnitReadWorkflowInitialization; + WorkflowReadInitialization brAPITrialReadWorkflowInitialization; + WorkflowReadInitialization brAPIStudyReadWorkflowInitialization; + WorkflowReadInitialization locationReadWorkflowInitialization; + WorkflowReadInitialization brAPIDatasetReadWorkflowInitialization; + WorkflowReadInitialization brAPIGermplasmReadWorkflowInitialization; + BrAPIReadFactory brAPIReadFactory; + + @Inject + public WorkflowInitialization(BrAPIReadFactory brAPIReadFactory) { + this.brAPIReadFactory = brAPIReadFactory; + } + @Override + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { + brAPIObservationUnitReadWorkflowInitialization = brAPIReadFactory.observationUnitWorkflowReadInitializationBean(context); + brAPITrialReadWorkflowInitialization = brAPIReadFactory.trialWorkflowReadInitializationBean(context); + brAPIStudyReadWorkflowInitialization = brAPIReadFactory.studyWorkflowReadInitializationBean(context); + locationReadWorkflowInitialization = brAPIReadFactory.locationWorkflowReadInitializationBean(context); + brAPIDatasetReadWorkflowInitialization = brAPIReadFactory.datasetWorkflowReadInitializationBean(context); + brAPIGermplasmReadWorkflowInitialization = brAPIReadFactory.germplasmWorkflowReadInitializationBean(context); + + log.debug("reading required BrAPI data from BrAPI service"); + try { + brAPIObservationUnitReadWorkflowInitialization.execute(); + brAPITrialReadWorkflowInitialization.execute(); + brAPIStudyReadWorkflowInitialization.execute(); + locationReadWorkflowInitialization.execute(); + brAPIDatasetReadWorkflowInitialization.execute(); + brAPIGermplasmReadWorkflowInitialization.execute(); + } catch (ApiException e) { + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); + return this.compensate(context); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java new file mode 100644 index 000000000..0893df9f2 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java @@ -0,0 +1,107 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; + +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; + +@Prototype +public class AppendStatistic { + private HashSet environmentNames; + private HashSet observationUnitIds; + private HashSet gids; + private int newCount; + private int existingCount; + private int mutatedCount; + + public AppendStatistic() { + this.clearData(); + } + + public void clearData() { + this.environmentNames = new HashSet<>(); + this.observationUnitIds = new HashSet<>(); + this.gids = new HashSet<>(); + this.newCount = 0; + this.existingCount = 0; + this.mutatedCount = 0; + } + public int incrementNewCount(Integer value) { + int increment = 0; + if (value == null) { + increment = 1; + } else if (value >= 0) { + increment = value; + } + this.newCount += increment; + + return this.newCount; + } + public int incrementExistingCount(Integer value) { + int increment = 0; + if (value == null) { + increment = 1; + } else if (value >= 0) { + increment = value; + } + this.existingCount += increment; + + return this.existingCount; + } + public int incrementMutatedCount(Integer value) { + int increment = 0; + if (value == null) { + increment = 1; + } else if (value >= 0) { + increment = value; + } + this.mutatedCount += increment; + + return this.mutatedCount; + } + public void addEnvironmentName(String name) { + Optional.ofNullable(name).ifPresent(environmentNames::add); + } + public void addObservationUnitId(String id) { + Optional.ofNullable(id).ifPresent(observationUnitIds::add); + } + public void addGid(String gid) { + Optional.ofNullable(gid).ifPresent(gids::add); + } + public Map constructPreviewMap() { + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder().newObjectCount(environmentNames.size()).build(); + ImportPreviewStatistics observationUnitsStats = ImportPreviewStatistics.builder().newObjectCount(observationUnitIds.size()).build(); + ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder().newObjectCount(gids.size()).build(); + ImportPreviewStatistics newStats = ImportPreviewStatistics.builder().newObjectCount(newCount).build(); + ImportPreviewStatistics existingStats = ImportPreviewStatistics.builder().newObjectCount(existingCount).build(); + ImportPreviewStatistics mutatedStats = ImportPreviewStatistics.builder().newObjectCount(mutatedCount).build(); + + return Map.of( + "Environments", environmentStats, + "Observation_Units", observationUnitsStats, + "GIDs", gidStats, + "Observations", newStats, + "Existing_Observations", existingStats, + "Mutated_Observations", mutatedStats + ); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java new file mode 100644 index 000000000..8b7614dce --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -0,0 +1,381 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import com.google.gson.Gson; +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.ProcessedDataFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.VisitedObservationData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.validator.field.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationVariableService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.dao.db.tables.pojos.TraitEntity; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.utilities.Utilities; +import tech.tablesaw.api.Table; +import tech.tablesaw.columns.Column; + +import javax.inject.Inject; +import java.time.OffsetDateTime; +import java.util.*; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.*; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.MULTIPLE_EXP_TITLES; + +@Slf4j +@Prototype +public class ImportTableProcess extends AppendOverwriteMiddleware { + @Property(name = "brapi.server.reference-source") + String brapiReferenceSource; + StudyService studyService; + ObservationVariableService observationVariableService; + ObservationService observationService; + BrAPIObservationDAO brAPIObservationDAO; + ExperimentUtilities experimentUtil; + Gson gson; + FieldValidator fieldValidator; + AppendStatistic statistic; + ProcessedDataFactory processedDataFactory; + + @Inject + public ImportTableProcess(StudyService studyService, + ObservationVariableService observationVariableService, + BrAPIObservationDAO brAPIObservationDAO, + ObservationService observationService, + ExperimentUtilities experimentUtil, + FieldValidator fieldValidator, + AppendStatistic statistic, + ProcessedDataFactory processedDataFactory) { + this.studyService = studyService; + this.observationVariableService = observationVariableService; + this.brAPIObservationDAO = brAPIObservationDAO; + this.observationService = observationService; + this.experimentUtil = experimentUtil; + this.gson = new Gson(); + this.fieldValidator = fieldValidator; + this.statistic = statistic; + this.processedDataFactory = processedDataFactory; + } + + @Override + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { + log.debug("verifying traits listed in import"); + + // Get all the dynamic columns of the import + ImportUpload upload = context.getImportContext().getUpload(); + Table data = context.getImportContext().getData(); + String[] dynamicColNames = upload.getDynamicColumnNames(); + List> dynamicCols = data.columns(dynamicColNames); + + // Collect the columns for observation variable data + List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); + List varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toList()); + + // Collect the columns for observation timestamps + List> timestampCols = dynamicCols.stream().filter(col -> col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); + Set tsNames = timestampCols.stream().map(Column::name).collect(Collectors.toSet()); + + // Construct validation errors for any timestamp columns that don't have a matching variable column + List importRows = context.getImportContext().getImportRows(); + Optional.ofNullable(context.getAppendOverwriteWorkflowContext().getValidationErrors()).orElseGet(() -> { + context.getAppendOverwriteWorkflowContext().setValidationErrors(new ValidationErrors()); + return new ValidationErrors(); + }); + ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); + List tsValErrs = observationVariableService.validateMatchedTimestamps(Set.copyOf(varNames), timestampCols).orElse(new ArrayList<>()); + for (int i = 0; i < importRows.size(); i++) { + int rowNum = i; + tsValErrs.forEach(validationError -> validationErrors.addError(rowNum, validationError)); + } + + try { + // Stop processing the import if there are unmatched timestamp columns + if (tsValErrs.size() > 0) { + throw new UnprocessableEntityException("One or more timestamp columns do not have a matching observation variable"); + } + + //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval + Map> tsColByPheno = timestampCols.stream().collect(Collectors.toMap(col -> col.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), col -> col)); + + // Add the map to the context for use in processing import + context.getAppendOverwriteWorkflowContext().setTimeStampColByPheno(tsColByPheno); + + // Fetch the traits named in the observation variable columns + Program program = context.getImportContext().getProgram(); + List traits = observationVariableService.fetchTraitsByName(Set.copyOf(varNames), program); + + // Map trait by phenotype column name + Map traitByPhenoColName = traits.stream().collect( + Collectors.toMap( + trait -> trait.getObservationVariableName().toUpperCase(), // Use uppercase keys for case-insensitivity + trait -> trait, + (trait1, trait2) -> trait1, // Merge function + CaseInsensitiveMap::new // Supplier for creating a CaseInsensitiveMap + ) + ); + + // Sort the traits to match the order of the headers in the import file + List sortedTraits = experimentUtil.sortByField(varNames, new ArrayList<>(traits), TraitEntity::getObservationVariableName); + + // Get the pending observation dataset + PendingImportObject pendingTrial = ExperimentUtilities.getSingleEntryValue(context.getAppendOverwriteWorkflowContext().getTrialByNameNoScope()).orElseThrow(()->new UnprocessableEntityException(MULTIPLE_EXP_TITLES.getValue())); + String datasetName = String.format("Observation Dataset [%s-%s]", program.getKey(), pendingTrial.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString()); + PendingImportObject pendingDataset = context.getAppendOverwriteWorkflowContext().getObsVarDatasetByName().get(datasetName); + + // Add new phenotypes to the pending observation dataset list (NOTE: "obsVarName [programKey]" is used instead of obsVarDbId) + // TODO: Change to using actual dbIds as per the BrAPI spec, instead of namespaced obsVar names (was necessary for Breedbase) + List datasetObsVarDbIds = pendingDataset.getBrAPIObject().getData().stream().collect(Collectors.toList()); + List phenoDbIds = sortedTraits.stream().map(t->Utilities.appendProgramKey(t.getObservationVariableName(), program.getKey())).collect(Collectors.toList()); + phenoDbIds.removeAll(datasetObsVarDbIds); + pendingDataset.getBrAPIObject().getData().addAll(phenoDbIds); + + // Update pending status + if (ImportObjectState.EXISTING == pendingDataset.getState()) { + pendingDataset.setState(ImportObjectState.MUTATED); + } + + // Read any observation data stored for these traits + log.debug("fetching observation data stored for traits"); + Set ouDbIds = context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().values().stream().map(u -> u.getBrAPIObject().getObservationUnitDbId()).collect(Collectors.toSet()); + Set varDbIds = sortedTraits.stream().map(t->t.getObservationVariableDbId()).collect(Collectors.toSet()); + List observations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, varDbIds, program); + + // Construct helper lookup tables to use for hashing stored observation data + Map unitNameByDbId = context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, BrAPIObservationUnit::getObservationUnitName)); + Map variableNameByDbId = sortedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableDbId, Trait::getObservationVariableName)); + Map studyNameByDbId = context.getAppendOverwriteWorkflowContext().getStudyByNameNoScope().values().stream() + .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) + .map(PendingImportObject::getBrAPIObject) + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); + + // Hash stored observation data using a signature of unit, variable, and study names + Map observationByObsHash = observations.stream().collect(Collectors.toMap(o->{ + return observationService.getObservationHash(unitNameByDbId.get(o.getObservationUnitDbId()), + variableNameByDbId.get(o.getObservationVariableDbId()), + studyNameByDbId.get(o.getStudyDbId())); + }, o->o)); + + // Add the observation data map to the context for use in processing import + context.getAppendOverwriteWorkflowContext().setExistingObsByObsHash(observationByObsHash); + + // Build new pending observation data for each phenotype + Map> pendingObservationByHash = new HashMap<>(); + + // In case the user aborted an import, clear any old preview statistics before processing the import + statistic.clearData(); + + // Build pending import data maps for each row + for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { + Integer rowNum = i; + ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + + // Construct the pending import for the row + Optional.ofNullable(context.getImportContext().getMappedBrAPIImport()).orElseGet(() -> { + context.getImportContext().setMappedBrAPIImport(new HashMap<>()); + return new HashMap<>(); + }); + PendingImport mappedImportRow = context.getImportContext().getMappedBrAPIImport().getOrDefault(rowNum, new PendingImport()); + String unitId = row.getObsUnitID(); + mappedImportRow.setTrial(context.getAppendOverwriteWorkflowContext().getPendingTrialByOUId().get(unitId)); + mappedImportRow.setLocation(context.getAppendOverwriteWorkflowContext().getPendingLocationByOUId().get(unitId)); + mappedImportRow.setStudy(context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId)); + mappedImportRow.setObservationUnit(context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(unitId)); + mappedImportRow.setGermplasm(context.getAppendOverwriteWorkflowContext().getPendingGermplasmByOUId().get(unitId)); + + // Assemble the pending observation data for all phenotypes + for (Column column : phenotypeCols) { + String cellData = column.getString(rowNum); + + // Generate hash for looking up prior observation data + String studyName = context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getStudyName(); + String unitName = context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(unitId).getBrAPIObject().getObservationUnitName(); + String phenoColumnName = column.name(); + String observationHash = observationService.getObservationHash(unitName, phenoColumnName, studyName); + + // Get timestamp if associated column + var cell = new Object() { // mutable reference object to make timestamp accessible in anonymous methods + String timestamp = null; + }; + String tsColumnName = null; + if (tsColByPheno.containsKey(phenoColumnName)) { + cell.timestamp = tsColByPheno.get(phenoColumnName).getString(rowNum); + tsColumnName = tsColByPheno.get(phenoColumnName).name(); + + // If timestamp is not valid, add a validation error + fieldValidator.validateField(tsColumnName, cell.timestamp, null).ifPresent(err -> { + cell.timestamp = null; + validationErrors.addError(rowNum + 2, err); // +2 because of excel header row and 1-based row index + }); + + } + + VisitedObservationData processedData = null; + + // Is there prior observation data for this unit + var? + if (observationByObsHash.containsKey(observationHash)) { + + // Clone the prior observation + BrAPIObservation observation = gson.fromJson(gson.toJson(observationByObsHash.get(observationHash)), BrAPIObservation.class); + + // Is there a change to the prior data? + if ((!cellData.isBlank() && !cellData.equals(observation.getValue())) || (cell.timestamp != null && !observationService.parseDateTime(cell.timestamp).equals(observation.getObservationTimeStamp()))) { + + // Is prior data protected? + /** + * For preview purposes all data can be treated as overwritable, but data cannot be + * overwritten if changes are to be committed and the user has not chosen to overwrite + */ + boolean canOverwrite = !context.getImportContext().isCommit() || !"false".equals(row.getOverwrite() == null ? "false" : row.getOverwrite()); + + // Clone the trait + Trait changeTrait = gson.fromJson(gson.toJson(traitByPhenoColName.get(phenoColumnName)), Trait.class); + + // Create new instance of OverwrittenData + processedData = processedDataFactory.overwrittenDataBean(canOverwrite, + context.getImportContext().isCommit(), + unitId, + changeTrait, + phenoColumnName, + tsColumnName, + cellData, + cell.timestamp, + Optional.ofNullable(row.getOverwriteReason()).orElse(""), + observation, + context.getImportContext().getUser().getId(), + program); + } else { + + // create new instance of UnchangedData + processedData = processedDataFactory.unchangedDataBean(observation, program); + } + + // + } else if (!cellData.isBlank()) { + + // Clone the observation unit and trait + BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(row.getObsUnitID()).getBrAPIObject()), BrAPIObservationUnit.class); + Trait initialTrait = gson.fromJson(gson.toJson(traitByPhenoColName.get(phenoColumnName)), Trait.class); + + // create new instance of InitialData + processedData = processedDataFactory.initialDataBean(brapiReferenceSource, + context.getImportContext().isCommit(), + context.getAppendOverwriteWorkflowContext().getPendingGermplasmByOUId().get(unitId).getBrAPIObject().getGermplasmName(), + context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getBrAPIObject(), + cellData, + cell.timestamp, + phenoColumnName, + tsColumnName, + initialTrait, + row, + pendingTrial.getId(), + context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getId(), + UUID.fromString(unitId), + context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getSeasons().get(0), + observationUnit, + context.getImportContext().getUser(), + context.getImportContext().getProgram()); + } else { + // Clone the observation unit + BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(row.getObsUnitID()).getBrAPIObject()), BrAPIObservationUnit.class); + + processedData = processedDataFactory.emptyDataBean(brapiReferenceSource, + context.getImportContext().isCommit(), + context.getAppendOverwriteWorkflowContext().getPendingGermplasmByOUId().get(unitId).getBrAPIObject().getGermplasmName(), + context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getBrAPIObject(), + phenoColumnName, + pendingTrial.getId(), + context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getId(), + UUID.fromString(unitId), + context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getSeasons().get(0), + observationUnit, + context.getImportContext().getUser(), + context.getImportContext().getProgram() + ); + } + + // Validate processed data + processedData.getValidationErrors().ifPresent(errList -> errList.forEach(e -> validationErrors.addError(rowNum + 2, e))); // +2 to account for header row and excel file 1-based row index + + // Update import preview statistics and set in the context + processedData.updateTally(statistic); + statistic.addEnvironmentName(studyName); + // TODO: change null values to actual data + // TODO: change signature to take two args, studyName and unitName + statistic.addObservationUnitId(null); + statistic.addGid(context.getAppendOverwriteWorkflowContext().getPendingGermplasmByOUId().get(unitId).getBrAPIObject().getAccessionNumber()); + context.getAppendOverwriteWorkflowContext().setStatistic(statistic); + + // Construct a pending observation + Optional> pendingProcessedData = Optional.ofNullable(processedData.constructPendingObservation()); + + // Set the new pending observation in the pending import for the row + pendingProcessedData.ifPresent(observation -> mappedImportRow.getObservations().add(observation)); + + // Add pending observation to map + pendingProcessedData.ifPresent(observation -> pendingObservationByHash.put(observationHash, observation)); + } + + // Set the pending import for the row + context.getImportContext().getMappedBrAPIImport().put(rowNum, mappedImportRow); + } + + // Throw the total list of all validation errors for the import + if (validationErrors.hasErrors()) { + throw new ValidatorException(validationErrors); + } + + // Add the pending observation map to the context for use in processing the import + context.getAppendOverwriteWorkflowContext().setPendingObservationByHash(pendingObservationByHash); + + return processNext(context); + } catch (DoesNotExistException | ApiException | UnprocessableEntityException | ValidatorException e) { + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); + return this.compensate(context); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteMiddleware.java new file mode 100644 index 000000000..8e8478db5 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteMiddleware.java @@ -0,0 +1,39 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; + +/** + * ExpUnitMiddleware class extends Middleware class to handle compensating transactions in the context of ExpUnitMiddlewareContext. + */ +public abstract class AppendOverwriteMiddleware extends Middleware { + + /** + * Compensates for an error that occurred in the current local transaction, tagging the error and undoing the previous local transaction. + * + * @param context The context in which the compensation is to be performed. + * @return True if the prior local transaction was successfully compensated, false otherwise. + */ + @Override + public AppendOverwriteMiddlewareContext compensate(AppendOverwriteMiddlewareContext context) { + // tag an error if it occurred in this local transaction + context.getAppendOverwriteWorkflowContext().getProcessError().tag(this.getClass().getName()); + + // undo the prior local transaction + return compensatePrior(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteMiddlewareContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteMiddlewareContext.java new file mode 100644 index 000000000..da868325b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteMiddlewareContext.java @@ -0,0 +1,32 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; + +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; + +@Getter +@Setter +@Builder +public class AppendOverwriteMiddlewareContext { + + private ImportContext importContext; + private AppendOverwriteWorkflowContext appendOverwriteWorkflowContext; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java new file mode 100644 index 000000000..13d892722 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java @@ -0,0 +1,70 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; + +import lombok.Getter; +import lombok.Setter; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; +import org.breedinginsight.model.ProgramLocation; +import tech.tablesaw.columns.Column; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@Getter +@Setter +public class AppendOverwriteWorkflowContext { + // Cache maps keyed by existing observation unit ids + private Set referenceOUIds = new HashSet<>(); + private Map> pendingTrialByOUId = new HashMap<>(); + private Map> pendingStudyByOUId = new HashMap<>(); + private Map> pendingObsUnitByOUId = new HashMap<>(); + private Map> pendingObsDatasetByOUId = new HashMap<>(); + private Map> pendingLocationByOUId = new HashMap<>(); + private Map> pendingGermplasmByOUId = new HashMap<>(); + + // Processing statistics + private AppendStatistic statistic; + + // Exceptions + private MiddlewareException processError; + private ValidationErrors validationErrors; + + // Cache maps keyed by name without program scope + private Map> observationUnitByNameNoScope; + private Map> trialByNameNoScope; + private Map> studyByNameNoScope; + private Map> locationByName; + private Map> obsVarDatasetByName; + + // Other helpful cache maps + private Map> existingGermplasmByGID; + private Map> pendingObservationByHash; + private Map> timeStampColByPheno; + private Map existingObsByObsHash; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/Middleware.java new file mode 100644 index 000000000..0a730a7b2 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/Middleware.java @@ -0,0 +1,75 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; + +public abstract class Middleware { + + Middleware next; + Middleware prior; + + /** + * Builds chains of middleware objects. + */ + public static Middleware link(Middleware first, Middleware... chain) { + Middleware head = first; + for (Middleware nextInChain: chain) { + nextInChain.prior = head.getLastLink(); + head.getLastLink().next = nextInChain; + head = nextInChain; + } + return first; + } + + /** + * Subclasses will implement this local transaction. + */ + public abstract T process(T context); + /** + * Subclasses will implement this method to handle errors and possibly undo the local transaction. + */ + public abstract T compensate(T context); + /** + * Processes the next local transaction or ends traversing if we're at the + * last local transaction of the transaction. + */ + protected T processNext(T context) { + if (next == null) { + return context; + } + return (T) next.process(context); + } + + /** + * Runs the compensating local transaction for the prior local transaction or ends traversing if + * we're at the first local transaction of the transaction. + */ + protected T compensatePrior(T context) { + if (prior == null) { + return context; + } + return (T) prior.compensate(context); + } + + private Middleware getLastLink() { + return this.next == null ? this : this.next.getLastLink(); + } + + private Middleware getFirstLink() { + return this.prior == null ? this : this.prior.getFirstLink(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/MiddlewareException.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/MiddlewareException.java new file mode 100644 index 000000000..b35a5e246 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/MiddlewareException.java @@ -0,0 +1,41 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; + +import lombok.Getter; +import lombok.Setter; + +public class MiddlewareException { + @Getter + @Setter + String localTransactionName; + + @Getter + @Setter + Exception exception; + + public MiddlewareException(Exception exception) { + this.exception = exception; + } + + public void tag(String name) { + if (this.getLocalTransactionName() == null) { + this.setLocalTransactionName(name); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java new file mode 100644 index 000000000..28eb14e29 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -0,0 +1,35 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; + +import lombok.*; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.ProgramLocation; +import tech.tablesaw.columns.Column; + +import java.util.HashMap; +import java.util.Map; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class PendingData { + private Map> observationUnitByNameNoScope; + private Map> trialByNameNoScope; + private Map> studyByNameNoScope; + private Map> locationByName; + private Map> obsVarDatasetByName; + private Map> existingGermplasmByGID; + private Map> pendingObservationByHash; + private Map> timeStampColByPheno; + private Map existingObsByObsHash; + private ValidationErrors validationErrors; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java new file mode 100644 index 000000000..5d847dce2 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java @@ -0,0 +1,107 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.services.ImportStatusService; +import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; +import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.List; +import java.util.Optional; + +@Slf4j +@Getter +@Singleton +public class NewExperimentWorkflow implements ExperimentWorkflow { + private final ExperimentWorkflowNavigator.Workflow workflow; + private final ImportStatusService statusService; + private final Provider experimentProcessorProvider; + private final Provider processorManagerProvider; + + @Inject + public NewExperimentWorkflow(ImportStatusService statusService, + Provider experimentProcessorProvider, + Provider processorManagerProvider){ + this.statusService = statusService; + this.experimentProcessorProvider = experimentProcessorProvider; + this.processorManagerProvider = processorManagerProvider; + this.workflow = ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION; + } + + @Override + public Optional process(ImportServiceContext context) { + // Metadata about this workflow processing the context + ImportWorkflow workflow = ImportWorkflow.builder() + .id(getWorkflow().getId()) + .name(getWorkflow().getName()) + .build(); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .workflow(workflow) // attach metadata of this workflow to response + .importPreviewResponse(Optional.empty()) + .caughtException(Optional.empty()) + .build()); + + // Skip this workflow unless creating a new experiment + if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result with metadata for this workflow + if (context == null) { + return result; + } + + // Build and return the preview response + try { + ImportPreviewResponse successResponse; + + // TODO: replace ProcessorManager#process with new-experiment workflow process from BI-2132 + successResponse = processorManagerProvider.get().process(context.getBrAPIImports(), + List.of(experimentProcessorProvider.get()), + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); + result.ifPresent(importWorkflowResult -> importWorkflowResult.setImportPreviewResponse(Optional.of(successResponse))); + } catch (Exception e) { + result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.of(e))); + } + + return result; + } + + @Override + public int getOrder() { + return 1; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java new file mode 100644 index 000000000..72c0545b6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java @@ -0,0 +1,36 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import com.fasterxml.jackson.annotation.JsonValue; +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Value; +import lombok.extern.slf4j.Slf4j; + +import javax.annotation.PostConstruct; + +@Slf4j +public class ExpImportProcessConstants { + + public static final CharSequence COMMA_DELIMITER = ","; + public static final String TIMESTAMP_PREFIX = "TS:"; + public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; + public static String BRAPI_REFERENCE_SOURCE; + public static final String MIDNIGHT = "T00:00:00-00:00"; + + public enum ErrMessage { + MULTIPLE_EXP_TITLES("File contains more than one Experiment Title"), + MISSING_OBS_UNIT_ID_ERROR("Experimental entities are missing ObsUnitIDs"), + PREEXISTING_EXPERIMENT_TITLE("Experiment Title already exists"); + + private String value; + + ErrMessage(String value) { + this.value = value; + } + + @JsonValue + public String getValue() { + return value; + } + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ImportContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ImportContext.java new file mode 100644 index 000000000..0738d6a1e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ImportContext.java @@ -0,0 +1,28 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import tech.tablesaw.api.Table; + +import java.util.List; +import java.util.Map; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ImportContext { + private ImportUpload upload; + private List importRows; + private Map mappedBrAPIImport; + private Table data; + private Program program; + private User user; + private boolean commit; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java new file mode 100644 index 000000000..9afbf189b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java @@ -0,0 +1,13 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; + +import java.util.Map; + +@Data +@ToString +@NoArgsConstructor +public class ProcessedData { + private Map statistics; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java new file mode 100644 index 000000000..f85137e15 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -0,0 +1,115 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.BrAPIListTypes; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + +@Singleton +public class DatasetService { + private final BrAPIListDAO brAPIListDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public DatasetService(BrAPIListDAO brapiListDAO) { + this.brAPIListDAO = brapiListDAO; + } + /** + * Module: Dataset Utility + * + * This module provides utility functions for interacting with datasets using the BrAPI standards. + * It includes methods for fetching dataset details, creating new datasets, updating existing datasets, etc. + * Usage: This module can be used in various applications where handling BrAPI-compliant datasets is required. + */ + + /** + * Fetches dataset details by dataset ID and program + * + * This function fetches details of a dataset by its ID and associated program from a data source using the BrAPI standards. + * + * @param id The unique identifier of the dataset to fetch + * @param program The program object associated with the dataset + * @return BrAPIListDetails object containing the details of the dataset + * @throws ApiException if there is an issue with fetching the dataset details from the data source + */ + public Optional fetchDatasetById(String id, Program program) throws ApiException { + Optional dataSetDetails = Optional.empty(); + + // Retrieve existing dataset summaries based on program ID and external reference + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(id)); + + // Check if the existing dataset summaries are returned, throw exception if not + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("Existing dataset summary not returned from BrAPI server"); + } + + // Retrieve dataset details using the list DB ID from the existing dataset summary + dataSetDetails = Optional.ofNullable(brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult()); + + return dataSetDetails; + } + + /** + * Constructs a PendingImportObject for a BrAPIListDetails dataset. + * This method retrieves the external reference for the dataset from the existing list + * based on a specific reference source. It then creates a PendingImportObject for the dataset + * with the existing list and reference ID. + * + * @param dataset The BrAPIListDetails dataset for which to construct the PendingImportObject + * @param program + * @return A PendingImportObject containing the dataset with the existing list and reference ID + * @throws IllegalStateException if external references weren't found for the list + */ + public PendingImportObject constructPIOFromDataset(BrAPIListDetails dataset, Program program) { + // Get the external reference for the dataset from the existing list + BrAPIExternalReference xref = Utilities.getExternalReference(dataset.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) + .orElseThrow(() -> new IllegalStateException("External references weren't found for list (dbid): " + dataset.getListDbId())); + + // Create a PendingImportObject for the dataset with the existing list and reference ID + return new PendingImportObject(ImportObjectState.EXISTING, dataset, UUID.fromString(xref.getReferenceId())); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java new file mode 100644 index 000000000..8ed3aa17e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java @@ -0,0 +1,98 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +@Singleton +public class GermplasmService { + private final BrAPIGermplasmDAO germplasmDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + public GermplasmService(BrAPIGermplasmDAO germplasmDAO) { + this.germplasmDAO = germplasmDAO; + } + + /** + * Retrieves a list of BrAPI Germplasm objects based on the provided set of database IDs and a Program object. + * + * @param dbIds A Set of database IDs (strings) used to filter germplasm data retrieval. + * @param program The Program object representing the program associated with the germplasm data. + * @return A List of BrAPIGermplasm objects that match the provided database IDs and program. + * @throws ApiException If an error occurs during the retrieval process. + */ + public List fetchGermplasmByDbId(Set dbIds, Program program) throws ApiException { + List brapiGermplasm = null; + brapiGermplasm = germplasmDAO.getGermplasmsByDBID(dbIds, program.getId()); + return brapiGermplasm; + } + + /** + * This method constructs a PendingImportObject for a given BrAPI Germplasm. + * It retrieves the External Reference associated with the Germplasm and constructs a PendingImportObject with ImportObjectState set to EXISTING. + * + * @param brapiGermplasm The BrAPI Germplasm object for which the PendingImportObject needs to be constructed + * @return PendingImportObject A PendingImportObject containing the BrAPI Germplasm object and its External Reference + * @throws IllegalStateException if the External Reference for the Germplasm is not found + */ + public PendingImportObject constructPIOFromBrapiGermplasm(BrAPIGermplasm brapiGermplasm) { + // Initialize the PendingImportObject to null + PendingImportObject pio = null; + + // Retrieve the External Reference associated with the Germplasm from the Utilities class + BrAPIExternalReference xref = Utilities.getExternalReference(brapiGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + // Throw an exception if External Reference is not found + .orElseThrow(() -> new IllegalStateException("External references weren't found for germplasm (dbid): " + brapiGermplasm.getGermplasmDbId())); + + // Construct the PendingImportObject with ImportObjectState set to EXISTING and External Reference UUID + pio = new PendingImportObject<>(ImportObjectState.EXISTING, brapiGermplasm, UUID.fromString(xref.getReferenceId())); + + return pio; + } + + /** + * Retrieves the Germplasm ID from a PendingImportObject containing BrAPI Germplasm data. + * This method extracts the Germplasm ID (GID) from the Accession Number of the BrAPI Germplasm object within the PendingImportObject. + * + * @param pio a PendingImportObject that wraps BrAPI Germplasm data + * @return a String representing the Germplasm ID extracted from the Accession Number + */ + public String getGIDFromGermplasmPIO(PendingImportObject pio) { + String gid = null; + gid = pio.getBrAPIObject().getAccessionNumber(); + return gid; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java new file mode 100644 index 000000000..73466224f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java @@ -0,0 +1,109 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.ProgramLocationService; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; + +@Singleton +@Slf4j +public class LocationService { + private final ProgramLocationService programLocationService; + public LocationService(ProgramLocationService programLocationService) { + this.programLocationService = programLocationService; + } + + /** + * Fetches a list of ProgramLocation objects based on the provided locationDbIds and program ID. + * + * @param locationDbIds A set of locationDbIds used to query locations. + * @param program The Program object for which locations are being fetched. + * @return A list of ProgramLocation objects matching the given locationDbIds for the program. + * @throws ApiException if there is an issue with fetching the locations or if any location(s) are not found. + */ + public List fetchLocationsByDbId(Set locationDbIds, Program program) throws ApiException { + List programLocations = null; // Initializing the ProgramLocations list + + // Retrieving locations based on the locationDbId and the program's ID + programLocations = programLocationService.getLocationsByDbId(locationDbIds, program.getId()); + + // If no locations are found, throw an IllegalStateException with an error message + if (locationDbIds.size() != programLocations.size()) { + Set missingIds = new HashSet<>(locationDbIds); + missingIds.removeAll(programLocations.stream().map(ProgramLocation::getLocationDbId).collect(Collectors.toSet())); + throw new IllegalStateException("Location not found for location dbid(s): " + String.join(COMMA_DELIMITER, missingIds)); + } + + // Return the fetched ProgramLocations + return programLocations; + } + + /** + * Constructs a PendingImportObject of type ProgramLocation from a given BrAPI ProgramLocation. + * This method creates a new PendingImportObject with the state set to EXISTING and the BrAPI ProgramLocation as the source object. + * + * @param brapiLocation The BrAPI ProgramLocation from which the PendingImportObject should be constructed + * @return PendingImportObject The PendingImportObject created from the BrAPI ProgramLocation + */ + public PendingImportObject constructPIOFromBrapiLocation(ProgramLocation brapiLocation) { + return new PendingImportObject<>(ImportObjectState.EXISTING, brapiLocation, brapiLocation.getId()); + } + + // used by expunit workflow + public Map> initializeLocationByName( + Program program, + Map> studyByName) { + Map> locationByName = new HashMap<>(); + + List existingLocations = new ArrayList<>(); + if(studyByName.size() > 0) { + Set locationDbIds = studyByName.values() + .stream() + .map(study -> study.getBrAPIObject() + .getLocationDbId()) + .collect(Collectors.toSet()); + try { + existingLocations.addAll(programLocationService.getLocationsByDbId(locationDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + existingLocations.forEach(existingLocation -> locationByName.put( + existingLocation.getName(), + new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()) + ) + ); + return locationByName; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java new file mode 100644 index 000000000..a6ecac2c0 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -0,0 +1,184 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang3.StringUtils; +import org.brapi.v2.model.core.BrAPISeason; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Scale; +import org.breedinginsight.model.User; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.*; +import java.util.stream.Collectors; + +@Singleton +public class ObservationService { + private final ExperimentUtilities experimentUtilities; + + @Inject + public ObservationService(ExperimentUtilities experimentUtilities) { + this.experimentUtilities = experimentUtilities; + } + + public boolean validCategory(List categories, String value) { + Set categoryValues = categories.stream() + .map(category -> category.getValue().toLowerCase()) + .collect(Collectors.toSet()); + return categoryValues.contains(value.toLowerCase()); + } + public boolean validNumericRange(BigDecimal value, Scale validValues) { + // account for empty min or max in valid determination + return (validValues.getValidValueMin() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMin())) >= 0) && + (validValues.getValidValueMax() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMax())) <= 0); + } + public Optional validNumericValue(String value) { + BigDecimal number; + try { + number = new BigDecimal(value); + } catch (NumberFormatException e) { + return Optional.empty(); + } + return Optional.of(number); + } + + public boolean isBlankObservation(String value) { + return StringUtils.isBlank(value); + } + public boolean isNAObservation(String value){ + return value.equalsIgnoreCase("NA"); + } + public boolean validDateTimeValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } + + public boolean validDateValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } + public String getObservationHash(String observationUnitName, String variableName, String studyName) { + String concat = DigestUtils.sha256Hex(observationUnitName) + + DigestUtils.sha256Hex(variableName) + + DigestUtils.sha256Hex(StringUtils.defaultString(studyName)); + return DigestUtils.sha256Hex(concat); + } + + public OffsetDateTime parseDateTime(String dateString) { + // Try parsing as ISO-8601 + try { + return OffsetDateTime.parse(dateString); + } catch (DateTimeParseException e) { + // If ISO-8601 parsing fails, try YY-MM-DD format + try { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + LocalDate localDate = LocalDate.parse(dateString, formatter); + return localDate.atStartOfDay().atOffset(ZoneOffset.UTC); + } catch (DateTimeParseException ex) { + // If both parsing attempts fail, return null + return null; + } + } + } + + /** + * Constructs a new BrAPI observation based on the provided parameters. + * + * @param commit boolean value indicating whether the operation should be committed + * @param germplasmName the name of the germplasm associated with the observation + * @param variableName the name of the observation variable + * @param study the BrAPI study object to associate with the observation + * @param seasonDbId the unique identifier of the season for the observation + * @param obsUnit the BrAPI observation unit object + * @param value the value of the observation + * @param trialId the identifier of the trial associated with the observation + * @param studyId the identifier of the study associated with the observation + * @param obsUnitId the identifier of the observation unit associated with the observation + * @param observationId the identifier of the observation + * @param referenceSource the source of the reference + * @param user the User object representing the creator of the observation + * @param program the Program object associated with the observation + * @return a newly constructed BrAPIObservation object + */ + public BrAPIObservation constructNewBrAPIObservation(boolean commit, + String germplasmName, + String variableName, + BrAPIStudy study, + String seasonDbId, + BrAPIObservationUnit obsUnit, + String value, + UUID trialId, + UUID studyId, + UUID obsUnitId, + UUID observationId, + String referenceSource, + User user, + Program program) { + BrAPIObservation observation = new BrAPIObservation(); + observation.setGermplasmName(germplasmName); + + observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.STUDY_NAME, Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); + + observation.setObservationVariableName(variableName); + observation.setObservationUnitDbId(obsUnit.getObservationUnitDbId()); + observation.setObservationUnitName(obsUnit.getObservationUnitName()); + observation.setValue(value); + + // The BrApi server needs this. Breedbase does not. + BrAPISeason season = new BrAPISeason(); + season.setSeasonDbId(seasonDbId); + observation.setSeason(season); + + if (commit) { + Map createdBy = new HashMap<>(); + createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_ID, user.getId()); + createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_NAME, user.getName()); + observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_BY, createdBy); + observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_DATE, DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(OffsetDateTime.now())); + + observation.setExternalReferences(experimentUtilities.constructBrAPIExternalReferences(program, referenceSource, trialId, null, studyId, obsUnitId, observationId)); + } + return observation; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java new file mode 100644 index 000000000..ff5da2813 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -0,0 +1,191 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import io.micronaut.context.annotation.Property; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; + +@Singleton +public class ObservationUnitService { + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public ObservationUnitService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + } + + /** + * Retrieves a list of BrAPI (Breeding API) observation units by their database IDs for a given set of experimental unit IDs and program. + * + * This method queries the BrAPIObservationUnitDAO to retrieve BrAPI observation units based on the provided experimental unit IDs and program. + * If the database IDs of the retrieved BrAPI observation units do not match the provided experimental unit IDs, an IllegalStateException is thrown. + * The exception includes information on the missing observation unit database IDs. + * + * @param expUnitIds a set of experimental unit IDs for which to retrieve BrAPI observation units + * @param program the program for which to retrieve BrAPI observation units + * @return a list of BrAPIObservationUnit objects corresponding to the provided experimental unit IDs + * @throws ApiException if an error occurs during the retrieval of observation units + * @throws IllegalStateException if the retrieved observation units do not match the provided experimental unit IDs + */ + public List getObservationUnitsByDbId(Set expUnitIds, Program program) throws ApiException, IllegalStateException { + List brapiUnits = null; + + // Retrieve reference Observation Units based on IDs + brapiUnits = brAPIObservationUnitDAO.getObservationUnitsById(expUnitIds, program); + + // If no BrAPI units are found, throw an IllegalStateException with an error message + if (expUnitIds.size() != brapiUnits.size()) { + Set missingIds = new HashSet<>(expUnitIds); + + // Calculate missing IDs based on retrieved BrAPI units + missingIds.removeAll(brapiUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toSet())); + + // Throw exception with missing IDs information + throw new IllegalStateException("Observation unit not found for unit dbid(s): " + String.join(COMMA_DELIMITER, missingIds)); + } + + return brapiUnits; + } + + /** + * Constructs a PendingImportObject of type BrAPIObservationUnit from a given BrAPIObservationUnit object. + * This function is responsible for constructing an import object that represents an observation unit for the Deltabreed system, + * using a provided BrAPIObservationUnit object from a BrAPI source. + * + * @param unit the BrAPIObservationUnit object to be used for constructing the PendingImportObject + * @return a PendingImportObject of type BrAPIObservationUnit representing the imported observation unit + * @throws IllegalStateException if the external reference for the observation unit does not exist + */ + public PendingImportObject constructPIOFromBrapiUnit(BrAPIObservationUnit unit) { + final PendingImportObject[] pio = new PendingImportObject[]{null}; + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + // Get external reference for the Observation Unit + Optional unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource); + unitXref.ifPresentOrElse( + xref -> { + pio[0] = new PendingImportObject(ImportObjectState.EXISTING, unit, UUID.fromString(xref.getReferenceId())); + }, + () -> { + + // but throw an error if no unit ID + throw new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID"); + } + ); + return pio[0]; + } + + /** + * Maps pending observation units by their reference IDs. + * This function takes a list of pending import objects representing BrAPI observation units + * and constructs a map where the key is the external reference ID of the observation unit + * and the value is the pending import object itself. + * + * @param pios List of pending import objects for BrAPI observation units + * @return A map of pending observation units keyed by their external reference ID + */ + public Map> mapPendingUnitById(List> pios) { + Map> pendingUnitById = new HashMap<>(); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + for (PendingImportObject pio : pios) { + + // Get external reference for the Observation Unit + Optional xref = Utilities.getExternalReference(pio.getBrAPIObject().getExternalReferences(), deltaBreedOUSource); + pendingUnitById.put(xref.get().getReferenceId(),pio); + } + + return pendingUnitById; + } + + /** + * This method takes a list of PendingImportObject objects and a Program object as input + * and maps the PendingImportObject objects by their observation unit name without the program scope. + * + * @param pios A list of PendingImportObject objects to be processed + * @param program The Program object representing the scope to be removed from observation unit names + * @return A Map> mapping observation unit names without the program scope to the corresponding PendingImportObject objects + */ + public Map> mapPendingUnitByNameNoScope(List> pios, + Program program) { + Map> pendingUnitByNameNoScope = new HashMap<>(); + + for (PendingImportObject pio : pios) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pio.getBrAPIObject().getStudyName(), + program.getKey() + ); + String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pio.getBrAPIObject().getObservationUnitName(), + program.getKey() + ); + pendingUnitByNameNoScope.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName), pio); + } + + return pendingUnitByNameNoScope; + } + + /** + * Collects missing Observation Unit IDs from a set of reference IDs and a list of existing Observation Units. + * + * This function takes a Set of reference IDs and a List of existing Observation Units, filters out the Observation Units + * that have external references matching a specific source, and returns a List of missing Observation Unit IDs that are + * present in the reference IDs but not found in the existing Observation Units. + * + * @param referenceIds The Set of reference IDs representing all possible Observation Unit IDs to match against. + * @param existingUnits The List of existing Observation Units to compare against the reference IDs. + * @return A List of Observation Unit IDs that are missing from the existing Observation Units but present in the reference IDs. + */ + public List collectMissingOUIds(Set referenceIds, List existingUnits) { + List missingIds = new ArrayList<>(referenceIds); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + Set fetchedIds = existingUnits.stream() + .filter(unit ->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).isPresent()) + .map(unit->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).get().getReferenceId()) + .collect(Collectors.toSet()); + missingIds.removeAll(fetchedIds); + + return missingIds; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java new file mode 100644 index 000000000..ce43db2b4 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java @@ -0,0 +1,106 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import io.micronaut.http.HttpStatus; +import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import tech.tablesaw.columns.Column; +import org.breedinginsight.dao.db.tables.pojos.TraitEntity; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +@Singleton +public class ObservationVariableService { + private final OntologyService ontologyService; + + @Inject + public ObservationVariableService(OntologyService ontologyService) { + this.ontologyService = ontologyService; + } + + /** + * Fetches traits by name for the given set of variable names and program. + * + * This method fetches all stored traits for the specified program and filters them based on the set of variable names provided. + * It ensures that all requested observation variables are present and returns a list of matching traits. + * If any observation variables are missing, it throws an IllegalStateException with the missing variable names. + * + * @param varNames a set of variable names to fetch traits for + * @param program the program for which traits are fetched + * @return a list of traits filtered by the provided variable names + * @throws DoesNotExistException if the program or traits do not exist + * @throws IllegalStateException if any requested observation variables are missing + */ + public List fetchTraitsByName(Set varNames, Program program) throws DoesNotExistException, IllegalStateException { + List traits = null; + + // Fetch all stored traits for the program + List programTraits = ontologyService.getTraitsByProgramId(program.getId(), true); + + // Only keep traits that are in the set of names + List upperCaseVarNames = varNames.stream().map(String::toUpperCase).collect(Collectors.toList()); + traits = programTraits.stream().filter(e -> upperCaseVarNames.contains(e.getObservationVariableName().toUpperCase())).collect(Collectors.toList()); + + // If any requested observation variables are missing, throw an IllegalStateException + if (varNames.size() != traits.size()) { + Set missingVarNames = new HashSet<>(varNames); + missingVarNames.removeAll(traits.stream().map(TraitEntity::getObservationVariableName).collect(Collectors.toSet())); + throw new IllegalStateException("Observation variables not found for name(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingVarNames)); + } + + return traits; + } + + /** + * Validates that each timestamp column corresponds to a phenotype column. + * + * This method takes a Set of observationVariableNames and a List of timestamp columns, and checks + * if each timestamp column corresponds to a phenotype column by comparing it with the observationVariableNames. + * + * @param observationVariableNames A Set of observation variable names representing the phenotype columns. + * @param timestampCols A List of timestamp columns to be validated. + * + * @return An Optional containing a list of ValidationErrors if there are mismatches, or an empty + * Optional if all timestamp columns have corresponding phenotype columns. + */ + public Optional> validateMatchedTimestamps(Set observationVariableNames, + List> timestampCols) { + Optional> ve = Optional.empty(); + + // Check that each timestamp column corresponds to a phenotype column + List valErrs = timestampCols.stream() + .filter(col -> !(observationVariableNames.contains(col.name().replaceFirst(ExpImportProcessConstants.TIMESTAMP_REGEX, StringUtils.EMPTY)))) + .map(col -> new ValidationError(col.name().toString(), String.format("Timestamp column %s lacks corresponding phenotype column", col.name().toString()), HttpStatus.UNPROCESSABLE_ENTITY)) + .collect(Collectors.toList()); + + if (valErrs.size() > 0) { + ve = Optional.of(valErrs); + } + + return ve; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java new file mode 100644 index 000000000..06f6a3bbb --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -0,0 +1,256 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import io.micronaut.context.annotation.Property; +import io.reactivex.functions.Function; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPISeason; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; + +@Singleton +@Slf4j +public class StudyService { + private final Map seasonDbIdToYearCache = new HashMap<>(); + private final BrAPISeasonDAO brAPISeasonDAO; + private final BrAPIStudyDAO brAPIStudyDAO; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public StudyService(BrAPISeasonDAO brAPISeasonDAO, + BrAPIStudyDAO brAPIStudyDAO) { + this.brAPISeasonDAO = brAPISeasonDAO; + this.brAPIStudyDAO = brAPIStudyDAO; + } + + // TODO: used by both workflows + public PendingImportObject processAndCacheStudy( + BrAPIStudy existingStudy, + Program program, + Function getterFunction, + Map> studyMap) throws Exception { + PendingImportObject pendingStudy; + BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); + // map season dbid to year + String seasonDbId = existingStudy.getSeasons().get(0); // It is assumed that the study has only one season + if(StringUtils.isNotBlank(seasonDbId)) { + String seasonYear = seasonDbIdToYear(seasonDbId, program.getId()); + existingStudy.setSeasons(Collections.singletonList(seasonYear)); + } + pendingStudy = new PendingImportObject<>( + ImportObjectState.EXISTING, + (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), + UUID.fromString(xref.getReferenceId()) + ); + studyMap.put( + Utilities.removeProgramKeyAndUnknownAdditionalData(getterFunction.apply(existingStudy), program.getKey()), + pendingStudy + ); + return pendingStudy; + } + + /** + * Constructs a PendingImportObject containing a BrAPIStudy object based on the provided BrAPIStudy and Program. + * This function retrieves the external reference for the study and maps the season dbid to the corresponding year. + * + * @param brAPIStudy The BrAPIStudy object to construct the PendingImportObject from. + * @param program The Program object associated with the study. + * @return A PendingImportObject containing the formatted BrAPIStudy object. + * @throws IllegalStateException If the external reference for the study is not found. + */ + public PendingImportObject constructPIOFromBrapiStudy(BrAPIStudy brAPIStudy, Program program) { + // Retrieve external reference for the study + BrAPIExternalReference xref = Utilities.getExternalReference(brAPIStudy.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) + .orElseThrow(() -> new IllegalStateException("External references weren't found for study (dbid): " + brAPIStudy.getStudyDbId())); + + // Map season dbid to year + String seasonDbId = brAPIStudy.getSeasons().get(0); // It is assumed that the study has only one season + if(StringUtils.isNotBlank(seasonDbId)) { + String seasonYear = seasonDbIdToYear(seasonDbId, program.getId()); + brAPIStudy.setSeasons(Collections.singletonList(seasonYear)); + } + + // Create and return a PendingImportObject for the BrAPIStudy + return new PendingImportObject<>( + ImportObjectState.EXISTING, + (BrAPIStudy) Utilities.formatBrapiObjForDisplay(brAPIStudy, BrAPIStudy.class, program), + UUID.fromString(xref.getReferenceId()) + ); + } + + // TODO: used by both workflows + public String seasonDbIdToYear(String seasonDbId, UUID programId) { + String year = null; + // TODO: add season objects to redis cache then just extract year from those + // removing this for now here + //if (this.seasonDbIdToYearCache.containsKey(seasonDbId)) { // get it from cache if possible + // year = this.seasonDbIdToYearCache.get(seasonDbId); + //} else { + year = seasonDbIdToYearFromDatabase(seasonDbId, programId); + // this.seasonDbIdToYearCache.put(seasonDbId, year); + //} + return year; + } + + // TODO: used by both workflows + private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { + BrAPISeason season = null; + try { + season = this.brAPISeasonDAO.getSeasonById(seasonDbId, programId); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + } + Integer yearInt = (season == null) ? null : season.getYear(); + return (yearInt == null) ? "" : yearInt.toString(); + } + + public String yearToSeasonDbIdFromDatabase(String year, UUID programId) { + BrAPISeason targetSeason = null; + List seasons; + try { + seasons = this.brAPISeasonDAO.getSeasonsByYear(year, programId); + for (BrAPISeason season : seasons) { + if (null == season.getSeasonName() || season.getSeasonName().isBlank() || season.getSeasonName().equals(year)) { + targetSeason = season; + break; + } + } + if (targetSeason == null) { + BrAPISeason newSeason = new BrAPISeason(); + Integer intYear = null; + if( StringUtils.isNotBlank(year) ){ + intYear = Integer.parseInt(year); + } + newSeason.setYear(intYear); + newSeason.setSeasonName(year); + targetSeason = this.brAPISeasonDAO.addOneSeason(newSeason, programId); + } + + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + log.error(e.getResponseBody(), e); + } + + return (targetSeason == null) ? null : targetSeason.getSeasonDbId(); + } + + public List seasonsFromDatabase(String year, UUID programId) { + List seasons = null; + try { + seasons = this.brAPISeasonDAO.getSeasonsByYear(year, programId); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + } + + return seasons; + } + /** + * Fetches a list of BrAPI studies by their study database IDs for a given program. + * + * This method queries the BrAPIStudyDAO to retrieve studies based on the provided study database IDs and the program. + * It ensures that all requested study database IDs are found in the result set, throwing an IllegalStateException if any are missing. + * + * @param studyDbIds a Set of Strings representing the study database IDs to fetch + * @param program the Program object representing the program context in which to fetch studies + * @return a List of BrAPIStudy objects matching the provided study database IDs + * + * @throws ApiException if there is an issue fetching the studies + * @throws IllegalStateException if any requested study database IDs are not found in the result set + */ + public List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { + List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); + if (studies.size() != studyDbIds.size()) { + List missingIds = new ArrayList<>(studyDbIds); + missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); + throw new IllegalStateException( + "Study not found for studyDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + return studies; + } + + /** + * Fetch BrAPI studies by their database identifiers for a given program. + * + * This method retrieves a list of BrAPI studies based on the provided set of study database identifiers + * and a specified program. It utilizes the BrAPIStudyDAO to fetch studies from the database. + * + * @param studyDbIds A set of study database identifiers for filtering the studies. + * @param program The program related to the studies. + * @return A list of BrAPIStudy objects representing the fetched studies. + * @throws ApiException If there are issues in retrieving studies or if any study database identifier is missing. + */ + public List fetchBrapiStudiesByDbId(Set studyDbIds, Program program) throws ApiException { + List brapiStudies = null; // Initializing the study object + brapiStudies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); // Retrieving studies from the database + + // If no studies are found, throw an IllegalStateException with an error message + if (studyDbIds.size() != brapiStudies.size()) { + Set missingIds = new HashSet<>(studyDbIds); + missingIds.removeAll(brapiStudies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toSet())); + throw new IllegalStateException("Study not found for location dbid(s): " + String.join(COMMA_DELIMITER, missingIds)); + } + + return brapiStudies; + } + + /** + * Retrieves the study database ID belonging to a pending unit in BrAPI format. + * + * This method takes a PendingImportObject containing a BrAPIObservationUnit + * object and returns the study database ID associated with the unit, if it exists. + * + * @param pio The PendingImportObject containing the BrAPIObservationUnit object for which the study database ID is to be retrieved. + * @return The study database ID belonging to the pending unit, or null if the unit does not exist or if the study database ID is not set. + */ + public String getStudyDbIdBelongingToPendingUnit(PendingImportObject pio) { + String studyDbId = null; + + // Check if the BrAPI object in the PendingImportObject is not null + if (pio.getBrAPIObject() != null) { + // Retrieve the study database ID from the BrAPIObservationUnit object + studyDbId = pio.getBrAPIObject().getStudyDbId(); + } + + return studyDbId; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java new file mode 100644 index 000000000..dcbc79159 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -0,0 +1,370 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; + +@Singleton +@Slf4j +public class TrialService { + private final BrAPITrialDAO brAPITrialDAO; + + private final StudyService studyService; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public TrialService(BrAPITrialDAO brAPITrialDAO, + StudyService studyService) { + this.brAPITrialDAO = brAPITrialDAO; + this.studyService = studyService; + } + /** + * Module: BrAPITrialService + * Description: This module contains methods for retrieving BrAPI trials based on trial database IDs and programs. + * brAPITrialDAO: Data Access Object for interacting with BrAPI trials in the database. + * fetchBrapiTrialsByDbId: Method to fetch BrAPI trials based on provided trial database IDs and program. + */ + + /** + * Retrieves the TrialDbId belonging to a pending unit based on the provided BrAPI observation unit and program. + * If the TrialDbId is directly assigned to the unit, it is returned. Otherwise, the TrialDbId + * assigned to the study belonging to the unit is retrieved. + * + * @param pendingUnit The pending import object representing the BrAPI observation unit. + * @param program The program associated with the pending import. + * @return The TrialDbId belonging to the pending unit. + * @throws IllegalStateException if TrialDbId and StudyDbId are not set for an existing ObservationUnit. + * @throws InternalServerException if there is an internal server error while fetching the TrialDbId. + */ + public String getTrialDbIdBelongingToPendingUnit(PendingImportObject pendingUnit, + Program program) throws IllegalStateException, InternalServerException { + String trialDbId = null; + + BrAPIObservationUnit brapiUnit = pendingUnit.getBrAPIObject(); + if (StringUtils.isBlank(brapiUnit.getTrialDbId()) && StringUtils.isBlank(brapiUnit.getStudyDbId())) { + throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); + } + + // get the trial directly assigned to the unit + if (StringUtils.isNotBlank(brapiUnit.getTrialDbId())) { + trialDbId = brapiUnit.getTrialDbId(); + } else { + + // or get the trial directly assigned to the study belonging to the unit + String studyDbId = brapiUnit.getStudyDbId(); + try { + trialDbId = getTrialDbIdBelongingToStudy(studyDbId, program); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + return trialDbId; + } + + /** + * Fetches BrAPI trials belonging to the specified trial database IDs and program. + * + * This method retrieves a list of BrAPI trials that are associated with the provided set of trial database IDs + * within the context of the given program. It ensures that all specified trial IDs are found in the result + * to maintain data integrity. + * + * @param trialDbIds A set of trial database IDs indicating the trials to fetch. + * @param program The program object representing the program to which the trials belong. + * @return A list of BrAPITrial objects representing the fetched trials. + * @throws IllegalStateException If not all specified trial database IDs are found in the fetched trials. + * @throws InternalServerException If an error occurs during the API call to fetch the trials. + */ + public List fetchBrapiTrialsBelongingToUnits(Set trialDbIds, Program program) { + try { + List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); + if (trials.size() != trialDbIds.size()) { + List missingIds = new ArrayList<>(trialDbIds); + missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); + throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + + return trials; + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + /** + * Retrieves a list of BrAPI trials based on a set of trial database IDs and a specified program. + * + * @param trialDbIds a set of trial database IDs used to retrieve the BrAPI trials + * @param program the program associated with the trials + * @return a list of BrAPITrial objects that match the provided trial database IDs and program + * @throws InternalServerException if there is an internal server error during the retrieval process + * @throws ApiException if there is an exception while fetching the trials + */ + public List fetchBrapiTrialsByDbId(Set trialDbIds, Program program) throws InternalServerException, ApiException { + // Initialize the list of BrAPI trials + List brapiTrials = null; + + // Retrieve the trials from the DAO based on the provided trial database IDs and program + brapiTrials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); + + // Check if all requested trials were found + if (trialDbIds.size() != brapiTrials.size()) { + // Identify the missing trial database IDs + Set missingIds = new HashSet<>(trialDbIds); + missingIds.removeAll(brapiTrials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toSet())); + + // Throw an exception with the list of missing trial database IDs + throw new IllegalStateException("Trial not found for trial dbid(s): " + String.join(COMMA_DELIMITER, missingIds)); + } + + // Return the list of retrieved BrAPI trials + return brapiTrials; + } + + + // TODO: also used in other workflow + public void initializeTrialsForExistingObservationUnits(ImportContext importContext, + PendingData pendingData) { + if(pendingData.getObservationUnitByNameNoScope().size() > 0) { + Set trialDbIds = new HashSet<>(); + Set studyDbIds = new HashSet<>(); + + pendingData.getObservationUnitByNameNoScope().values() + .forEach(pio -> { + BrAPIObservationUnit existingOu = pio.getBrAPIObject(); + if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { + throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); + } + + if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { + trialDbIds.add(existingOu.getTrialDbId()); + } else { + studyDbIds.add(existingOu.getStudyDbId()); + } + }); + + //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId + if(!studyDbIds.isEmpty()) { + try { + trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, importContext.getProgram())); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + try { + List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, importContext.getProgram()); + if (trials.size() != trialDbIds.size()) { + List missingIds = new ArrayList<>(trialDbIds); + missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); + throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + + trials.forEach(trial -> processAndCacheTrial(trial, importContext.getProgram(), pendingData.getTrialByNameNoScope())); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + } + + /** + * Fetches trial DbIds for the given study DbIds by using the BrAPI studies API. + * + * @param studyDbIds The set of study DbIds for which to fetch trial DbIds. + * @param program The program associated with the studies. + * @return A set of trial DbIds corresponding to the provided study DbIds. + * @throws ApiException If there was an error while fetching the studies or if a study does not have a trial DbId. + * @throws IllegalStateException If the trial DbId is not set for an existing study. + */ + private Set fetchTrialDbidsForStudies(Set studyDbIds, Program program) throws ApiException { + Set trialDbIds = new HashSet<>(); + List studies = studyService.fetchStudiesByDbId(studyDbIds, program); + studies.forEach(study -> { + if (StringUtils.isBlank(study.getTrialDbId())) { + throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); + } + trialDbIds.add(study.getTrialDbId()); + }); + + return trialDbIds; + } + + /** + * Retrieves the trialDbId that belongs to a specific study with the given studyDbId and program. + * + * @param studyDbId The unique identifier for the study in the system. + * @param program The program object associated with the study. + * @return The trialDbId associated with the study. + * @throws ApiException if the study with the provided studyDbId is not found in the BrAPI service. + * @throws IllegalStateException if the trialDbId is not set for the existing study. + */ + private String getTrialDbIdBelongingToStudy(String studyDbId, Program program) throws ApiException { + String trialDbId = null; + List studies = studyService.fetchStudiesByDbId(Set.of(studyDbId), program); + if (studies.size() == 0) { + throw new ApiException("Study not found in BrAPI service: " + studyDbId); + } + BrAPIStudy study = studies.get(0); + if (StringUtils.isBlank(study.getTrialDbId())) { + throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); + } + trialDbId = study.getTrialDbId(); + + return trialDbId; + } + + /** + * This method processes an existing trial, retrieves the experiment ID from the trial's external references, + * and caches the trial with the corresponding experiment ID in a map. + * + * @param existingTrial The existing BrAPITrial object to be processed and cached. + * @param program The Program object associated with the trial. + * @param trialByNameNoScope The map to cache the trial by its name without program scope. (will be modified in place) + * + * @throws InternalServerException + */ + private void processAndCacheTrial( + BrAPITrial existingTrial, + Program program, + Map> trialByNameNoScope) { + + //get TrialId from existingTrial + BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) + .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); + UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); + + trialByNameNoScope.put( + Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), + new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); + } + + /** + * Constructs a PendingImportObject containing a BrAPITrial object based on the input BrAPITrial. + * + * This function takes a BrAPITrial object as input and constructs a PendingImportObject which + * encapsulates the trial along with its associated experiment ID. The experiment ID is retrieved + * from the external references of the trial object using utility method getExternalReference. + * If the experiment ID is not found in the external references, an InternalServerException is thrown. + * + * @param trial the BrAPITrial object for which the PendingImportObject is to be constructed + * @return a PendingImportObject containing the BrAPITrial object and its associated experiment ID + * @throws InternalServerException if the experiment ID is not found in the external references of the trial + */ + public PendingImportObject constructPIOFromBrapiTrial(BrAPITrial trial) throws InternalServerException { + PendingImportObject pio = null; + BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(trial.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) + .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); + UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); + pio = new PendingImportObject<>(ImportObjectState.EXISTING, trial, experimentId); + return pio; + } + + /** + * Initializes trials by name without scope for the given program. + * + * @param program the program to initialize trials for + * @param observationUnitByNameNoScope a map of observation units by name without scope + * @param experimentImportRows a list of experiment observation rows + * @return a map of trials by name with pending import objects + * + * @throws InternalServerException + */ + private Map> initializeTrialByNameNoScope(Program program, Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> trialByName = new HashMap<>(); + + initializeTrialsForExistingObservationUnits(program, observationUnitByNameNoScope, trialByName); + + List uniqueTrialNames = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID())) + .map(ExperimentObservation::getExpTitle) + .distinct() + .collect(Collectors.toList()); + try { + brAPITrialDAO.getTrialsByName(uniqueTrialNames, program).forEach(existingTrial -> + processAndCacheTrial(existingTrial, program, trialByName) + ); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + return trialByName; + } + + private void initializeTrialsForExistingObservationUnits(Program program, Map> observationUnitByNameNoScope, Map> trialByName) { + } + + // TODO: used by expunit workflow + public Map> mapPendingTrialByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> trialByName, + Map> studyByName, + Map> trialByOUId, + Program program + ) { + String trialName; + if (unit.getTrialName() != null) { + trialName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getTrialName(), program.getKey()); + } else if (unit.getStudyName() != null) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); + trialName = Utilities.removeProgramKeyAndUnknownAdditionalData( + studyByName.get(studyName).getBrAPIObject().getTrialName(), + program.getKey() + ); + } else { + throw new IllegalStateException("Observation unit missing trial name and study name: " + unitId); + } + trialByOUId.put(unitId, trialByName.get(trialName)); + + return trialByOUId; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java new file mode 100644 index 000000000..b69a4ddfe --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/DateValidator.java @@ -0,0 +1,102 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; + +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.model.Trait; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; +import static org.breedinginsight.dao.db.enums.DataType.DATE; + +/** + * This class represents a DateValidator that implements the ObservationValidator interface. + * It is responsible for validating date fields in observations based on specific criteria. + */ +@Slf4j +@Singleton +public class DateValidator implements ObservationValidator { + + @Inject + ObservationService observationService; + + private final String dateMessage = "Incorrect date format detected. Expected YYYY-MM-DD"; + private final String dateTimeMessage = "Incorrect datetime format detected. Expected YYYY-MM-DD or YYYY-MM-DDThh:mm:ss+hh:mm"; + + /** + * Constructor for the DateValidator class that initializes the ObservationService instance. + * @param observationService An instance of ObservationService used for validation. + */ + public DateValidator(ObservationService observationService) { + this.observationService = observationService; + } + + /** + * Validates a specific field in an observation based on the value and traits provided. + * + * @param fieldName The name of the field to validate. + * @param value The value of the field to be validated. + * @param variable The trait variable associated with the field. + * @return An Optional containing a ValidationError if validation fails, or Optional.empty() if validation passes. + */ + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + // Check if the observation is blank + if (observationService.isBlankObservation(value)) { + log.debug(String.format("Skipping validation of observation because there is no value.\n\tVariable: %s", fieldName)); + return Optional.empty(); + } + + // Check if the observation is NA + if (observationService.isNAObservation(value)) { + log.debug(String.format("Skipping validation of observation because it is NA.\n\tVariable: %s", fieldName)); + return Optional.empty(); + } + + // Check if the field is a timestamp + if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + if (!observationService.validDateValue(value) && !observationService.validDateTimeValue(value)) { + return Optional.of(new ValidationError(fieldName, dateTimeMessage, HttpStatus.UNPROCESSABLE_ENTITY)); + } + + } else { + // Skip if there is no trait data + if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { + return Optional.empty(); + } + + // Skip if this is not a date trait + if (!DATE.equals(variable.getScale().getDataType())) { + return Optional.empty(); + } + + // Validate date + if (!observationService.validDateValue(value)) { + return Optional.of(new ValidationError(fieldName, dateMessage, HttpStatus.UNPROCESSABLE_ENTITY)); + } + } + + return Optional.empty(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/FieldValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/FieldValidator.java new file mode 100644 index 000000000..d4ec36e01 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/FieldValidator.java @@ -0,0 +1,65 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; + +import io.micronaut.context.annotation.Primary; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.model.Trait; + +import javax.inject.Singleton; +import java.util.List; +import java.util.Optional; + +/** + * This class represents a FieldValidator that implements ObservationValidator interface to validate fields. + * FieldValidator is a Primary and Singleton bean in the application. + */ +@Primary +@Singleton +public class FieldValidator implements ObservationValidator { + + /** + * List of ObservationValidator instances to perform validation on fields. + */ + private final List validators; + + /** + * Constructor for FieldValidator which accepts a list of ObservationValidator instances. + * @param validators List of ObservationValidator instances + */ + public FieldValidator(List validators) { + this.validators = validators; + } + + /** + * Validates a field by applying validation from multiple validators in the list. + * Returns the first validation error encountered, if any. + * @param fieldName The name of the field being validated + * @param value The value of the field being validated + * @param variable The trait variable associated with the field + * @return Optional<ValidationError> Optional containing the first validation error encountered, if any + */ + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + return validators.stream() + .map(validator->validator.validateField(fieldName, value, variable)) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NominalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NominalValidator.java new file mode 100644 index 000000000..1bc0fa3e5 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NominalValidator.java @@ -0,0 +1,96 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; + +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.model.Trait; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; +import static org.breedinginsight.dao.db.enums.DataType.NOMINAL; + +/** + * This class represents a NominalValidator which implements the ObservationValidator interface. + * It is responsible for validating nominal fields within observations. + */ +@Slf4j +@Singleton +public class NominalValidator implements ObservationValidator { + + @Inject + ObservationService observationService; + + /** + * Constructor for NominalValidator class that takes an ObservationService as a parameter. + * @param observationService the ObservationService used for validation + */ + public NominalValidator(ObservationService observationService) { + this.observationService = observationService; + } + + /** + * Validates a field within an observation for nominal data. + * + * @param fieldName the name of the field being validated + * @param value the value of the field being validated + * @param variable the Trait variable associated with the field + * @return an Optional containing a ValidationError if validation fails, otherwise an empty Optional + */ + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + // Skip validation if observation is blank + if (observationService.isBlankObservation(value)) { + log.debug(String.format("Skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + // Skip validation if observation is NA + if (observationService.isNAObservation(value)) { + log.debug(String.format("Skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + // Skip if field is a timestamp + if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + return Optional.empty(); + } + + // Skip if there is no trait data + if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { + return Optional.empty(); + } + + // Skip if this is not a nominal trait + if (!NOMINAL.equals(variable.getScale().getDataType())) { + return Optional.empty(); + } + + // Validate categories + if (!observationService.validCategory(variable.getScale().getCategories(), value)) { + return Optional.of(new ValidationError(fieldName, "Undefined nominal category detected", HttpStatus.UNPROCESSABLE_ENTITY)); + } + + return Optional.empty(); + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java new file mode 100644 index 000000000..5fb3b9480 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/NumericalValidator.java @@ -0,0 +1,106 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; + +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.model.Trait; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.math.BigDecimal; +import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; +import static org.breedinginsight.dao.db.enums.DataType.NUMERICAL; + +/** + * NumericalValidator class is responsible for validating numerical observations against specified constraints. + * This class implements the ObservationValidator interface. + * + * The class provides a validateField method which takes the field name, value, and trait as inputs and + * returns an Optional containing a ValidationError if any validation error is encountered, or Optional.empty() if + * the validation passes successfully. + */ +@Slf4j +@Singleton +public class NumericalValidator implements ObservationValidator { + + @Inject + ObservationService observationService; + + /** + * Constructor for NumericalValidator class. + * @param observationService An instance of ObservationService to perform observation-related operations. + */ + public NumericalValidator(ObservationService observationService) { + this.observationService = observationService; + } + + /** + * Validates a numerical observation against specified constraints. + * Returns an Optional containing a ValidationError if validation fails, or Optional.empty() if successful. + * @param fieldName The name of the observation field to be validated. + * @param value The value of the observation to be validated. + * @param variable The trait associated with the observation. + * @return Optional containing a ValidationError if validation fails, or Optional.empty() if successful. + */ + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + if (observationService.isBlankObservation(value)) { + log.debug(String.format("Skipping validation of observation because there is no value.\n\tVariable: %s", fieldName)); + return Optional.empty(); + } + + if (observationService.isNAObservation(value)) { + log.debug(String.format("Skipping validation of observation because it is NA.\n\tVariable: %s", fieldName)); + return Optional.empty(); + } + + // Skip validation if field is a timestamp + if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + return Optional.empty(); + } + + // Skip validation if there is no trait data or if the trait is not numerical + if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null || + !NUMERICAL.equals(variable.getScale().getDataType())) { + return Optional.empty(); + } + + // Check if the value is a valid numeric value + Optional number = observationService.validNumericValue(value); + if (number.isEmpty()) { + return Optional.of(new ValidationError(fieldName, "Non-numeric text in a numerical field", HttpStatus.UNPROCESSABLE_ENTITY)); + } + + // Perform range validation for numeric value + Optional validationError = number + .flatMap(num -> { + if (observationService.validNumericRange(num, variable.getScale())) { + return Optional.empty(); // Return empty Optional if value is within numeric range + } else { + return Optional.of(new ValidationError(fieldName, "Value outside of min/max range detected", HttpStatus.UNPROCESSABLE_ENTITY)); + } + }); + + return validationError; + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/ObservationValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/ObservationValidator.java new file mode 100644 index 000000000..6a5bb5e10 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/ObservationValidator.java @@ -0,0 +1,30 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; + +import io.micronaut.core.annotation.NonNull; +import io.micronaut.core.order.Ordered; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.model.Trait; + +import java.util.Optional; + +@FunctionalInterface +public interface ObservationValidator extends Ordered { + Optional validateField(@NonNull String fieldName, @NonNull String value, Trait variable); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/OrdinalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/OrdinalValidator.java new file mode 100644 index 000000000..e961f6871 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/validator/field/OrdinalValidator.java @@ -0,0 +1,95 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.validator.field; + +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.model.Trait; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; +import static org.breedinginsight.dao.db.enums.DataType.ORDINAL; + +/** + * This class represents a validator specifically designed for ordinal traits. + * It implements the ObservationValidator interface. + */ +@Slf4j +@Singleton +public class OrdinalValidator implements ObservationValidator { + + @Inject + ObservationService observationService; + + /** + * Constructs an instance of OrdinalValidator with the specified ObservationService. + * + * @param observationService the ObservationService used for validation + */ + public OrdinalValidator(ObservationService observationService) { + this.observationService = observationService; + } + + /** + * Validates a field for ordinal traits. + * + * @param fieldName the name of the field being validated + * @param value the value of the field + * @param variable the trait related to the field + * @return an Optional containing a ValidationError if validation fails, empty otherwise + */ + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + if (observationService.isBlankObservation(value)) { + log.debug(String.format("Skipping validation of observation because there is no value.\n\tVariable: %s", fieldName)); + return Optional.empty(); + } + + if (observationService.isNAObservation(value)) { + log.debug(String.format("Skipping validation of observation because it is NA.\n\tVariable: %s", fieldName)); + return Optional.empty(); + } + + // Skip validation if field is a timestamp + if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + return Optional.empty(); + } + + // Skip validation if there is no trait data + if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { + return Optional.empty(); + } + + // Skip validation if this is not an ordinal trait + if (!ORDINAL.equals(variable.getScale().getDataType())) { + return Optional.empty(); + } + + // Validate categories + if (!observationService.validCategory(variable.getScale().getCategories(), value)) { + return Optional.of(new ValidationError(fieldName, "Undefined ordinal category detected", HttpStatus.UNPROCESSABLE_ENTITY)); + } + + return Optional.empty(); + } +} diff --git a/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java b/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java index d231116ae..5de4e7344 100644 --- a/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java +++ b/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java @@ -122,7 +122,9 @@ public void populate(@NotNull UUID key) { } finally { log.debug("Releasing semaphore: " + cacheKey); connection.getAtomicLong(cacheKey+":refreshing").set(0); + log.debug("Fetched connection: " + cacheKey); semaphore.release(); + log.debug("connection released"); } }); } diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 6bc203d64..7171155d0 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -24,9 +24,11 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.client.RxHttpClient; import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.netty.cookies.NettyCookie; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.typeAdapters.PaginationTypeAdapter; import org.brapi.v2.model.BrAPIExternalReference; @@ -77,16 +79,16 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import static io.micronaut.http.HttpRequest.GET; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.*; +@Slf4j @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class ExperimentFileImportTest extends BrAPITest { - private static final String OVERWRITE = "overwrite"; - private FannyPack securityFp; private String mappingId; private BiUserEntity testUser; @@ -146,6 +148,8 @@ public class ExperimentFileImportTest extends BrAPITest { (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .registerTypeAdapter(BrAPIPagination.class, new PaginationTypeAdapter()) .create(); + private String newExperimentWorkflowId; + private String appendOverwriteWorkflowId; @BeforeAll public void setup() { @@ -155,6 +159,11 @@ public void setup() { testUser = (BiUserEntity) setupObjects.get("testUser"); securityFp = (FannyPack) setupObjects.get("securityFp"); + /** + * Implicit test that the workflow ids are assigned in the following order + */ + newExperimentWorkflowId = importTestUtils.getExperimentWorkflowId(client, 0); + appendOverwriteWorkflowId = importTestUtils.getExperimentWorkflowId(client, 1); } /* @@ -176,6 +185,7 @@ public void setup() { @Test @SneakyThrows public void importNewExpNewLocNoObsSuccess() { + log.debug("importNewExpNewLocNoObsSuccess"); Program program = createProgram("New Exp and Loc", "NEXPL", "NEXPL", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); Map validRow = new HashMap<>(); validRow.put(Columns.GERMPLASM_GID, "1"); @@ -194,16 +204,20 @@ public void importNewExpNewLocNoObsSuccess() { validRow.put(Columns.COLUMN, "1"); validRow.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String workflowId = "new-experiment"; + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId, newExperimentWorkflowId); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + // TODO: remove this + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); @@ -217,6 +231,7 @@ public void importNewExpNewLocNoObsSuccess() { @Test @SneakyThrows public void importNewExpMultiNewEnvSuccess() { + log.debug("importNewExpMultiNewEnvSucces"); Program program = createProgram("New Exp and Multi New Env", "MULENV", "MULENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); Map firstEnv = new HashMap<>(); firstEnv.put(Columns.GERMPLASM_GID, "1"); @@ -252,16 +267,18 @@ public void importNewExpMultiNewEnvSuccess() { secondEnv.put(Columns.COLUMN, "1"); secondEnv.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(2, previewRows.size()); JsonObject firstRow = previewRows.get(0).getAsJsonObject(); @@ -283,6 +300,7 @@ public void importNewExpMultiNewEnvSuccess() { @Test @SneakyThrows public void importExistingExpAndEnvErrorMessage() { + log.debug("importExistingExpAndEnvErrorMessage"); Program program = createProgram("New Env Existing Exp", "DUPENV", "DUPENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); @@ -299,7 +317,8 @@ public void importExistingExpAndEnvErrorMessage() { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + JsonObject expResult = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + //JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); Map dupExp = new HashMap<>(); dupExp.put(Columns.GERMPLASM_GID, "1"); @@ -316,21 +335,23 @@ public void importExistingExpAndEnvErrorMessage() { dupExp.put(Columns.ROW, "1"); dupExp.put(Columns.COLUMN, "1"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, false, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + expResult = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, false, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, expResult.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + expResult); + assertTrue(expResult.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); } @Test @SneakyThrows public void importNewEnvNoObsSuccess() { + log.debug("importNewEnvNoObsSuccess"); Program program = createProgram("New Env", "NEWENV", "NEWENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); Map newEnv = new HashMap<>(); @@ -348,9 +369,10 @@ public void importNewEnvNoObsSuccess() { newEnv.put(Columns.ROW, "1"); newEnv.put(Columns.COLUMN, "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); @@ -365,6 +387,7 @@ public void importNewEnvNoObsSuccess() { @ValueSource(booleans = {true, false}) @SneakyThrows public void verifyMissingDataThrowsError(boolean commit) { + log.debug("verifyMissingDataThrowsError"); Program program = createProgram("Missing Req Cols "+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); Map base = new HashMap<>(); @@ -382,48 +405,59 @@ public void verifyMissingDataThrowsError(boolean commit) { Map noGID = new HashMap<>(base); noGID.remove(Columns.GERMPLASM_GID); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit, newExperimentWorkflowId); Map noExpTitle = new HashMap<>(base); noExpTitle.remove(Columns.EXP_TITLE); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit, newExperimentWorkflowId); Map noExpUnit = new HashMap<>(base); noExpUnit.remove(Columns.EXP_UNIT); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit, newExperimentWorkflowId); Map noExpType = new HashMap<>(base); noExpType.remove(Columns.EXP_TYPE); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit, newExperimentWorkflowId); Map noEnv = new HashMap<>(base); noEnv.remove(Columns.ENV); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit, newExperimentWorkflowId); Map noEnvLoc = new HashMap<>(base); noEnvLoc.remove(Columns.ENV_LOCATION); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); Map noExpUnitId = new HashMap<>(base); noExpUnitId.remove(Columns.EXP_UNIT_ID); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit, newExperimentWorkflowId); Map noExpRep = new HashMap<>(base); noExpRep.remove(Columns.REP_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit, newExperimentWorkflowId); Map noExpBlock = new HashMap<>(base); noExpBlock.remove(Columns.BLOCK_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit, newExperimentWorkflowId); Map noEnvYear = new HashMap<>(base); noEnvYear.remove(Columns.ENV_YEAR); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); } @Test @SneakyThrows public void importNewExpWithObsVar() { + log.debug("importNewExpWithObsVar"); List traits = importTestUtils.createTraits(1); Program program = createProgram("New Exp with Observations Vars", "EXPVRR", "EXPVRR", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -442,7 +476,8 @@ public void importNewExpWithObsVar() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -461,6 +496,7 @@ public void importNewExpWithObsVar() { @ValueSource(booleans = {true, false}) @SneakyThrows public void verifyDiffYearSameEnvThrowsError(boolean commit) { + log.debug("verifyDiffYEarSameEnvThrowsError"); Program program = createProgram("Diff Years "+(commit ? "C" : "P"), "YEARS"+(commit ? "C" : "P"), "YEARS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(2), null); List> rows = new ArrayList<>(); @@ -492,13 +528,16 @@ public void verifyDiffYearSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); + } @ParameterizedTest @ValueSource(booleans = {true, false}) @SneakyThrows public void verifyDiffLocSameEnvThrowsError(boolean commit) { + log.debug("verifyDiffLocSameEnvThrowsError"); Program program = createProgram("Diff Locations "+(commit ? "C" : "P"), "LOCS"+(commit ? "C" : "P"), "LOCS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(2), null); List> rows = new ArrayList<>(); @@ -530,13 +569,15 @@ public void verifyDiffLocSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); } @ParameterizedTest @ValueSource(booleans = {true, false}) @SneakyThrows public void importNewExpWithObs(boolean commit) { + log.debug("importNewExpWithObs"); List traits = importTestUtils.createTraits(1); Program program = createProgram("New Exp with Observations "+(commit ? "C" : "P"), "NEXOB"+(commit ? "C" : "P"), "NEXOB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -555,7 +596,8 @@ public void importNewExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -577,6 +619,7 @@ public void importNewExpWithObs(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { + log.debug("verifyFailureImportNewExpWithInvalidObs"); List traits = importTestUtils.createTraits(1); Program program = createProgram("Invalid Observations "+(commit ? "C" : "P"), "INVOB"+(commit ? "C" : "P"), "INVOB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -595,13 +638,16 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "Red"); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); + } @ParameterizedTest @ValueSource(booleans = {true, false}) @SneakyThrows public void verifyFailureNewOuExistingEnv(boolean commit) { + log.debug("verifyFailureNewOuExistingEnv"); Program program = createProgram("New OU Existing Env "+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); @@ -618,21 +664,24 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + //importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); Map newOU = new HashMap<>(newExp); newOU.put(Columns.EXP_UNIT_ID, "a-2"); newOU.put(Columns.ROW, "1"); newOU.put(Columns.COLUMN, "2"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, true, client, program, mappingId, newExperimentWorkflowId); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); @@ -641,6 +690,7 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { @Test @SneakyThrows public void importNewObsVarExistingOu() { + log.debug("importNewObsVarExistingOu"); List traits = importTestUtils.createTraits(2); Program program = createProgram("New ObsVar Existing OU", "OUVAR", "OUVAR", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -659,7 +709,7 @@ public void importNewObsVarExistingOu() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -687,7 +737,7 @@ public void importNewObsVarExistingOu() { newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -705,6 +755,7 @@ public void importNewObsVarExistingOu() { @Test @SneakyThrows public void importNewObsVarByObsUnitId() { + log.debug("importNewObsVarByObsUnitId"); List traits = importTestUtils.createTraits(2); Program program = createProgram("New ObsVar Referring to OU by ID", "OUVAR", "VAROU", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -723,7 +774,7 @@ public void importNewObsVarByObsUnitId() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -738,7 +789,7 @@ public void importNewObsVarByObsUnitId() { newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -757,6 +808,7 @@ public void importNewObsVarByObsUnitId() { @ValueSource(booleans = {true, false}) @SneakyThrows public void importNewObservationDataByObsUnitId(boolean commit) { + log.debug("importNewObservationDataByObsUnitId"); List traits = importTestUtils.createTraits(1); Program program = createProgram("New Observation Referring to OU by ID"+(commit ? "C" : "P"), "OUDAT"+(commit ? "C" : "P"), "DATOU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -775,7 +827,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); // empty dataset - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -803,7 +855,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -827,6 +879,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void importNewObsExistingOu(boolean commit) { + log.debug("importNewObsExistingOu"); List traits = importTestUtils.createTraits(1); Program program = createProgram("New Obs Existing OU "+(commit ? "C" : "P"), "OUOBS"+(commit ? "C" : "P"), "OUOBS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -844,7 +897,7 @@ public void importNewObsExistingOu(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -872,7 +925,7 @@ public void importNewObsExistingOu(boolean commit) { newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -893,6 +946,7 @@ public void importNewObsExistingOu(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { + log.debug("verifyFailureImportNewObsExistingOuWithExistingObs"); List traits = importTestUtils.createTraits(1); Program program = createProgram("New Obs Existing Obs "+(commit ? "C" : "P"), "FEXOB"+(commit ? "C" : "P"), "FEXOB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -911,7 +965,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -939,7 +993,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "2"); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); } /* @@ -948,9 +1002,11 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { - a new experiment is created after the first experiment - verify the second experiment gets created successfully */ + //TODO: this one @Test @SneakyThrows public void importSecondExpAfterFirstExpWithObs() { + log.debug("importSecondExpAfterFirstExpWithObs"); List traits = importTestUtils.createTraits(1); Program program = createProgram("New Exp After First", "NEAF", "NEAF", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExpA = new HashMap<>(); @@ -969,7 +1025,8 @@ public void importSecondExpAfterFirstExpWithObs() { newExpA.put(Columns.COLUMN, "1"); newExpA.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + //JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + JsonObject resultA = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsA = resultA.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsA.size()); @@ -997,7 +1054,8 @@ public void importSecondExpAfterFirstExpWithObs() { newExpB.put(Columns.COLUMN, "1"); newExpB.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + //JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + JsonObject resultB = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsB = resultB.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsB.size()); @@ -1020,6 +1078,7 @@ public void importSecondExpAfterFirstExpWithObs() { @ValueSource(booleans = {true, false}) @SneakyThrows public void importNewObsAfterFirstExpWithObs(boolean commit) { + log.debug("importNewObsAfterFirstExpWithObs"); List traits = importTestUtils.createTraits(2); Program program = createProgram("Exp with additional Uploads "+(commit ? "C" : "P"), "EXAU"+(commit ? "C" : "P"), "EXAU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -1030,7 +1089,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newExp.put(Columns.EXP_TYPE, "Phenotyping"); newExp.put(Columns.ENV, "New Env"); newExp.put(Columns.ENV_LOCATION, "Location A"); - newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.ENV_YEAR, "2025"); newExp.put(Columns.EXP_UNIT_ID, "a-1"); newExp.put(Columns.REP_NUM, "1"); newExp.put(Columns.BLOCK_NUM, "1"); @@ -1038,7 +1097,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1057,7 +1116,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(Columns.EXP_TYPE, "Phenotyping"); newObservation.put(Columns.ENV, "New Env"); newObservation.put(Columns.ENV_LOCATION, "Location A"); - newObservation.put(Columns.ENV_YEAR, "2023"); + newObservation.put(Columns.ENV_YEAR, "2025"); newObservation.put(Columns.EXP_UNIT_ID, "a-1"); newObservation.put(Columns.REP_NUM, "1"); newObservation.put(Columns.BLOCK_NUM, "1"); @@ -1067,7 +1126,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -1094,6 +1153,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { + log.debug("importNewObsAfterFirstExpWithObs_blank"); List traits = importTestUtils.createTraits(2); Program program = createProgram("Exp with additional Uploads (blank) "+(commit ? "C" : "P"), "EXAUB"+(commit ? "C" : "P"), "EXAUB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -1112,7 +1172,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1143,7 +1203,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newObservation.put(traits.get(0).getObservationVariableName(), ""); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -1154,10 +1214,12 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); + Map bothPhenotypeObservations = new HashMap<>(newObservation); + bothPhenotypeObservations.put(traits.get(0).getObservationVariableName(), "1"); if(commit) { - assertRowSaved(newObservation, program, traits); + assertRowSaved(bothPhenotypeObservations, program, traits); } else { - assertValidPreviewRow(newObservation, row, program, traits); + assertValidPreviewRow(bothPhenotypeObservations, row, program, traits); } } @@ -1525,6 +1587,33 @@ private JsonObject uploadAndVerifyFailure(Program program, File file, String exp JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); + assertEquals(1, rowErrors.size()); + JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); + assertEquals(1, fieldErrors.size()); + JsonObject error = fieldErrors.get(0).getAsJsonObject(); + assertEquals(expectedColumnError, error.get("field").getAsString()); + assertEquals(422, error.get("httpStatusCode").getAsInt()); + + return result; + } + + private JsonObject uploadAndVerifyWorkflowFailure(Program program, File file, String expectedColumnError, boolean commit, String workflowId) throws InterruptedException, IOException { + + //Flowable> call = importTestUtils.uploadDataFile(file, null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(file, null, true, client, program, mappingId, newExperimentWorkflowId); + //JsonObject result = JsonParser.parseString(upload).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); assertEquals(1, rowErrors.size()); JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); diff --git a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java index 35fe42946..30bb1830d 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java @@ -81,6 +81,7 @@ public class GermplasmFileImportTest extends BrAPITest { (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .create(); + @BeforeAll public void setup() { importTestUtils = new ImportTestUtils(); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index 12b79ac15..1c7b4acbb 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -28,6 +28,7 @@ import io.micronaut.http.client.multipart.MultipartBody; import io.micronaut.http.netty.cookies.NettyCookie; import io.reactivex.Flowable; +import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.model.v1.request.ProgramRequest; import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.api.v1.controller.TestTokenValidator; @@ -57,6 +58,7 @@ * * To use, instantiate a new instance of this class, then use it like a regular static utility class */ +@Slf4j public class ImportTestUtils { Pattern UUID_REGEX = Pattern.compile("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"); @@ -97,6 +99,38 @@ public Flowable> uploadDataFile(File file, Map> uploadWorkflowDataFile(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) { + + MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); + + // Upload file + String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", program.getId(), mappingId); + Flowable> call = client.exchange( + POST(uploadUrl, requestBody) + .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + String importId = result.get("importId").getAsString(); + + // Process data + String url = String.format("/programs/%s/import/mappings/%s/workflows/%s/data/%s/%s", program.getId(), mappingId, workflowId, importId, commit ? "commit" : "preview"); + Flowable> processCall = client.exchange( + PUT(url, userData) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + return processCall; + + } + public HttpResponse getUploadedFile(String importId, RxHttpClient client, Program program, String mappingId) throws InterruptedException { Flowable> call = client.exchange( GET(String.format("/programs/%s/import/mappings/%s/data/%s?mapping=true", program.getId(), mappingId, importId)) @@ -107,6 +141,7 @@ public HttpResponse getUploadedFile(String importId, RxHttpClient client if (response.getStatus().equals(HttpStatus.ACCEPTED)) { Thread.sleep(1000); + log.debug("202 Accepted response. Sleeping for 1000ms."); return getUploadedFile(importId, client, program, mappingId); } else { return response; @@ -155,7 +190,34 @@ public Map setup(RxHttpClient client, Gson gson, DSLContext dsl, "securityFp", securityFp); } + /** + * TODO: assumes new workflow is first in list, doesn't look at position property, would be more robust to + * look at that instead of assuming order + * @return + */ + public String getExperimentWorkflowId(RxHttpClient client, int workflowIndex) { + // Get the mapping id for experiment imports + Flowable> mappingCall = client.exchange( + GET("/import/mappings?importName=ExperimentsTemplateMap").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class); + HttpResponse mappingResponse = mappingCall.blockingFirst(); + String mappingId = JsonParser.parseString(mappingResponse.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + // Get the workflow id for the workflow at position workflowIndex in the collection of all available experiment workflows + // GET /import/mappings{?importName} + Flowable> call = client.exchange( + GET("/import/mappings/"+mappingId+"/workflows").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + return JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(workflowIndex).getAsJsonObject().get("id").getAsString(); + } public JsonObject uploadAndFetch(File file, Map userData, Boolean commit, RxHttpClient client, Program program, String mappingId) throws InterruptedException { Flowable> call = uploadDataFile(file, userData, commit, client, program, mappingId); @@ -170,6 +232,43 @@ public JsonObject uploadAndFetch(File file, Map userData, Boolea return result; } + public JsonObject uploadAndFetchWorkflow(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { + Flowable> call = uploadWorkflowDataFile(file, userData, commit, client, program, mappingId, workflowId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + return result; + } + + public JsonObject uploadAndFetchWorkflowNoStatusCheck(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { + Flowable> call = uploadWorkflowDataFile(file, userData, commit, client, program, mappingId, workflowId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + return result; + } + public List createTraits(int numToCreate) { List traits = new ArrayList<>(); for (int i = 0; i < numToCreate; i++) { diff --git a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java index 8c5b308c3..71bf6197c 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java @@ -28,6 +28,7 @@ import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.typeAdapters.PaginationTypeAdapter; import org.brapi.v2.model.BrAPIExternalReference; @@ -81,6 +82,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +@Slf4j @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @@ -146,6 +148,8 @@ public class SampleSubmissionFileImportTest extends BrAPITest { .registerTypeAdapter(BrAPIPagination.class, new PaginationTypeAdapter()) .create(); + private String newExperimentWorkflowId; + @BeforeAll public void setup() { importTestUtils = new ImportTestUtils(); @@ -153,7 +157,7 @@ public void setup() { mappingId = (String) setupObjects.get("mappingId"); testUser = (BiUserEntity) setupObjects.get("testUser"); securityFp = (FannyPack) setupObjects.get("securityFp"); - + newExperimentWorkflowId = importTestUtils.getExperimentWorkflowId(client, 0); } /* @@ -169,6 +173,7 @@ public void setup() { @Test @SneakyThrows public void importGIDSuccess() { + log.debug("importGIDSuccess"); Program program = createProgram("Import GID Success", "GIDS", "GIDS", BRAPI_REFERENCE_SOURCE, createGermplasm(96), null); List> validFile = new ArrayList<>(); @@ -215,6 +220,7 @@ public void importGIDSuccess() { @Test @SneakyThrows public void importObsUnitIdSuccess() { + log.debug("importObsUnitIdSuccess"); Program program = createProgram("Import ObsUnitID success", "OBSID", "OBSID", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); var experimentId = createExperiment(program); @@ -265,6 +271,7 @@ public void importObsUnitIdSuccess() { @ValueSource(booleans = {true, false}) @SneakyThrows public void importMissingGIDAndObsUnitIdFailure(boolean commit) { + log.debug("importMissingGIDAndObsUnitIdFailure"); Program program = createProgram("Missing GID/ObsUnit ID " + (commit ? "C" : "P"), "MGIOB"+ (commit ? "C" : "P"), "MGIOB"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, null, null); List> validFile = new ArrayList<>(); @@ -288,6 +295,7 @@ public void importMissingGIDAndObsUnitIdFailure(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void verifyMissingDataThrowsError(boolean commit) { + log.debug("verifyMissingDataThrowsError"); Program program = createProgram("Missing Req Cols "+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(96), null); Map base = new HashMap<>(); base.put(Columns.PLATE_ID, "valid_1"); @@ -312,6 +320,7 @@ public void verifyMissingDataThrowsError(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void importInvalidGIDFailure(boolean commit) { + log.debug("importInvalidGIDFailure"); Program program = createProgram("Invalid GID " + (commit ? "C" : "P"), "INGID"+ (commit ? "C" : "P"), "INGID"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, null, null); List> validFile = new ArrayList<>(); @@ -335,6 +344,7 @@ public void importInvalidGIDFailure(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void importInvalidObsUnitIdFailure(boolean commit) { + log.debug("importInvalidObsUnitIdFailure"); Program program = createProgram("Invalid ObsUnit ID " + (commit ? "C" : "P"), "INOBS"+ (commit ? "C" : "P"), "INOBS"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, null, null); List> validFile = new ArrayList<>(); @@ -358,6 +368,7 @@ public void importInvalidObsUnitIdFailure(boolean commit) { @ValueSource(booleans = {true, false}) @SneakyThrows public void importConflictingWellsFailure(boolean commit) { + log.debug("importConflictingWellsFailure"); Program program = createProgram("Conflicting Wells " + (commit ? "C" : "P"), "WELL"+ (commit ? "C" : "P"), "WELL"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(2), null); List> validFile = new ArrayList<>(); @@ -556,13 +567,14 @@ private String createExperiment(Program program) throws IOException, Interrupted .getAsJsonArray("data") .get(0).getAsJsonObject().get("id").getAsString(); - JsonObject importResult = importTestUtils.uploadAndFetch( + JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null), null, true, client, program, - expMappingId); + expMappingId, + newExperimentWorkflowId); return importResult .get("preview").getAsJsonObject() .get("rows").getAsJsonArray()