From 79189ba768ad3219c57ea950b04a543409e6b77f Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 18 Jun 2024 14:18:23 -0400 Subject: [PATCH 01/21] Messing with dependency issue --- .../experimentObservation/ExperimentImportService.java | 9 +++++++++ .../steps/PopulateNewPendingImportObjectsStep.java | 8 +------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 149da0e77..013d55012 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.model.imports.DomainImportService; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; @@ -26,12 +27,14 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import java.util.List; @Singleton @Slf4j public class ExperimentImportService extends DomainImportService { private final String IMPORT_TYPE_ID = "ExperimentImport"; + private final ExperimentWorkflowNavigator workflowNavigator; // TODO: delete processor fields once WorkflowNavigator is used @Inject @@ -40,6 +43,7 @@ public ExperimentImportService(Provider experimentProcessor ExperimentWorkflowNavigator workflowNavigator) { super(experimentProcessorProvider, processorManagerProvider, workflowNavigator); + this.workflowNavigator = workflowNavigator; } @Override @@ -51,5 +55,10 @@ public ExperimentObservation getImportClass() { public String getImportTypeId() { return IMPORT_TYPE_ID; } + + @Override + public List getWorkflows() throws Exception{ + return workflowNavigator.getWorkflows(); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index a471b84f3..f5ec2cc96 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -73,9 +73,7 @@ @Slf4j public class PopulateNewPendingImportObjectsStep { - private final ExperimentValidateService experimentValidateService; private final ExperimentSeasonService experimentSeasonService; - private final BrAPIObservationDAO brAPIObservationDAO; private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final DSLContext dsl; private final Gson gson; @@ -84,14 +82,10 @@ public class PopulateNewPendingImportObjectsStep { private String BRAPI_REFERENCE_SOURCE; @Inject - public PopulateNewPendingImportObjectsStep(ExperimentValidateService experimentValidateService, - ExperimentSeasonService experimentSeasonService, - BrAPIObservationDAO brAPIObservationDAO, + public PopulateNewPendingImportObjectsStep(ExperimentSeasonService experimentSeasonService, BrAPIObservationUnitDAO brAPIObservationUnitDAO, DSLContext dsl) { - this.experimentValidateService = experimentValidateService; this.experimentSeasonService = experimentSeasonService; - this.brAPIObservationDAO = brAPIObservationDAO; this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.dsl = dsl; this.gson = new JSON().getGson(); From 0ef8cd5d21a559ea886683b3109dcea422acd147 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 14:57:51 -0400 Subject: [PATCH 02/21] Removed ununsed fileImportService injection and resolved DI issue --- .../brapps/importer/services/FileImportService.java | 5 +++++ .../brapps/importer/services/FileMappingUtil.java | 8 +------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 0c2a9309d..da5a68e10 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -576,6 +576,11 @@ public List getWorkflowsForSystemMapping(UUID mappingId) throws .orElseThrow(() -> new DoesNotExistException("Cannot find mapping config associated with upload.")); BrAPIImportService importService = configManager.getImportServiceById(mappingConfig.getImportTypeId()) .orElseThrow(() -> new DoesNotExistException("Config with that id does not exist")); + // NOTE: + // this is creating a workflow navigator to call getWorkflows + // workflowNavigator.getWorkflows(); + // getWorkflows is creating the ExperimentWorkflow which is injecting the file import service and + // resulting in a circular dependency return importService.getWorkflows(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java index cb93daf48..8d3ed8146 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java @@ -32,13 +32,7 @@ @Singleton public class FileMappingUtil { - public static final String EXPERIMENT_TEMPLATE_NAME = "ExperimentsTemplateMap"; - private FileImportService fileImportService; - - - @Inject - public FileMappingUtil(FileImportService fileImportService) { - this.fileImportService = fileImportService; + public FileMappingUtil() { } // Returns a list of integers to identify the target row of the relationship. -1 if no relationship was found From 1acd04bd343865d81edbf02d2af56b05bf04f8e7 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:23:25 -0400 Subject: [PATCH 03/21] Added in BI-2128 change --- .../workflow/steps/PopulateNewPendingImportObjectsStep.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index f5ec2cc96..ddb4193ab 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -567,7 +567,9 @@ private void fetchOrCreateObservationPIO(ProcessedPhenotypeData phenotypeData, key = getImportObservationHash(importRow, variableName); if (existingObsByObsHash.containsKey(key)) { - if (!isObservationMatched(phenotypeData, pendingData, key, value, column, rowNum)){ + // NOTE: BI-2128 change added after refactor branch + // Update observation value only if it is changed and new value is not blank. + if (!isObservationMatched(phenotypeData, pendingData, key, value, column, rowNum) && StringUtils.isNotBlank(value)){ // prior observation with updated value newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); From 6a4043d2202adfa7c97e06d65bbc4d14c5045fc3 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:36:42 -0400 Subject: [PATCH 04/21] Updated create only tests to use create workflow --- .../importer/ExperimentFileImportTest.java | 239 ++++++++++++------ .../brapps/importer/ImportTestUtils.java | 95 ++++++- 2 files changed, 243 insertions(+), 91 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 48bffd4c2..fbb02435e 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -24,6 +24,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.client.RxHttpClient; import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.netty.cookies.NettyCookie; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; import lombok.SneakyThrows; @@ -77,6 +78,7 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import static io.micronaut.http.HttpRequest.GET; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.*; @@ -142,9 +144,11 @@ public class ExperimentFileImportTest extends BrAPITest { private BrAPISeasonDAO seasonDAO; private Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) - (json, type, context) -> OffsetDateTime.parse(json.getAsString())) - .registerTypeAdapter(BrAPIPagination.class, new PaginationTypeAdapter()) - .create(); + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .registerTypeAdapter(BrAPIPagination.class, new PaginationTypeAdapter()) + .create(); + + private String newExperimentWorkflowId; @BeforeAll public void setup() { @@ -153,7 +157,27 @@ public void setup() { mappingId = (String) setupObjects.get("mappingId"); testUser = (BiUserEntity) setupObjects.get("testUser"); securityFp = (FannyPack) setupObjects.get("securityFp"); + newExperimentWorkflowId = getNewExperimentWorkflowId(); + } + + /** + * TODO: assumes new workflow is first in list, doesn't look at position property, would be more robust to + * look at that instead of assuming order + * @return + */ + public String getNewExperimentWorkflowId() { + // GET /import/mappings{?importName} + Flowable> call = client.exchange( + GET("/import/mappings/"+mappingId+"/workflows").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + + return JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); } /* @@ -193,16 +217,20 @@ public void importNewExpNewLocNoObsSuccess() { validRow.put(Columns.COLUMN, "1"); validRow.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String workflowId = "new-experiment"; + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId, newExperimentWorkflowId); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + // TODO: remove this + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); @@ -251,16 +279,18 @@ public void importNewExpMultiNewEnvSuccess() { secondEnv.put(Columns.COLUMN, "1"); secondEnv.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(2, previewRows.size()); JsonObject firstRow = previewRows.get(0).getAsJsonObject(); @@ -298,7 +328,8 @@ public void importExistingExpAndEnvErrorMessage() { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + JsonObject expResult = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + //JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); Map dupExp = new HashMap<>(); dupExp.put(Columns.GERMPLASM_GID, "1"); @@ -315,16 +346,17 @@ public void importExistingExpAndEnvErrorMessage() { dupExp.put(Columns.ROW, "1"); dupExp.put(Columns.COLUMN, "1"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, false, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + expResult = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, false, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, expResult.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + expResult); + assertTrue(expResult.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); } @Test @@ -347,9 +379,10 @@ public void importNewEnvNoObsSuccess() { newEnv.put(Columns.ROW, "1"); newEnv.put(Columns.COLUMN, "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); @@ -381,43 +414,53 @@ public void verifyMissingDataThrowsError(boolean commit) { Map noGID = new HashMap<>(base); noGID.remove(Columns.GERMPLASM_GID); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit, newExperimentWorkflowId); Map noExpTitle = new HashMap<>(base); noExpTitle.remove(Columns.EXP_TITLE); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit, newExperimentWorkflowId); Map noExpUnit = new HashMap<>(base); noExpUnit.remove(Columns.EXP_UNIT); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit, newExperimentWorkflowId); Map noExpType = new HashMap<>(base); noExpType.remove(Columns.EXP_TYPE); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit, newExperimentWorkflowId); Map noEnv = new HashMap<>(base); noEnv.remove(Columns.ENV); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit, newExperimentWorkflowId); Map noEnvLoc = new HashMap<>(base); noEnvLoc.remove(Columns.ENV_LOCATION); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); Map noExpUnitId = new HashMap<>(base); noExpUnitId.remove(Columns.EXP_UNIT_ID); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit, newExperimentWorkflowId); Map noExpRep = new HashMap<>(base); noExpRep.remove(Columns.REP_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit, newExperimentWorkflowId); Map noExpBlock = new HashMap<>(base); noExpBlock.remove(Columns.BLOCK_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit, newExperimentWorkflowId); Map noEnvYear = new HashMap<>(base); noEnvYear.remove(Columns.ENV_YEAR); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); } @Test @@ -441,7 +484,8 @@ public void importNewExpWithObsVar() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -491,7 +535,9 @@ public void verifyDiffYearSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); + } @ParameterizedTest @@ -529,7 +575,8 @@ public void verifyDiffLocSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); } @ParameterizedTest @@ -554,7 +601,8 @@ public void importNewExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -594,7 +642,9 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "Red"); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); + } @ParameterizedTest @@ -617,21 +667,24 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + //importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); Map newOU = new HashMap<>(newExp); newOU.put(Columns.EXP_UNIT_ID, "a-2"); newOU.put(Columns.ROW, "1"); newOU.put(Columns.COLUMN, "2"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, true, client, program, mappingId, newExperimentWorkflowId); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); @@ -1022,6 +1075,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { - a new experiment is created after the first experiment - verify the second experiment gets created successfully */ + //TODO: this one @Test @SneakyThrows public void importSecondExpAfterFirstExpWithObs() { @@ -1043,7 +1097,8 @@ public void importSecondExpAfterFirstExpWithObs() { newExpA.put(Columns.COLUMN, "1"); newExpA.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + //JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + JsonObject resultA = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsA = resultA.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsA.size()); @@ -1071,7 +1126,8 @@ public void importSecondExpAfterFirstExpWithObs() { newExpB.put(Columns.COLUMN, "1"); newExpB.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + //JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + JsonObject resultB = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsB = resultB.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsB.size()); @@ -1337,10 +1393,10 @@ private Map assertRowSaved(Map expected, Program assertEquals(expected.get(Columns.GERMPLASM_GID), germplasm.getAccessionNumber()); if(expected.containsKey(Columns.TEST_CHECK) && StringUtils.isNotBlank((String)expected.get(Columns.TEST_CHECK))) { assertEquals(expected.get(Columns.TEST_CHECK), - ou.getObservationUnitPosition() - .getEntryType() - .name() - .substring(0, 1)); + ou.getObservationUnitPosition() + .getEntryType() + .name() + .substring(0, 1)); } assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); @@ -1433,10 +1489,10 @@ private Map assertValidPreviewRow(Map expected, if(traits != null) { assertNotNull(actual.get("observations")); observations = StreamSupport.stream(actual.getAsJsonArray("observations") - .spliterator(), false) - .map(obs -> gson.fromJson(obs.getAsJsonObject() - .getAsJsonObject("brAPIObject"), BrAPIObservation.class)) - .collect(Collectors.toList()); + .spliterator(), false) + .map(obs -> gson.fromJson(obs.getAsJsonObject() + .getAsJsonObject("brAPIObject"), BrAPIObservation.class)) + .collect(Collectors.toList()); ret.put("observations", observations); } @@ -1444,10 +1500,10 @@ private Map assertValidPreviewRow(Map expected, assertEquals(expected.get(Columns.GERMPLASM_GID), germplasm.getAccessionNumber()); if(expected.containsKey(Columns.TEST_CHECK) && StringUtils.isNotBlank((String)expected.get(Columns.TEST_CHECK))) { assertEquals(expected.get(Columns.TEST_CHECK), - ou.getObservationUnitPosition() - .getEntryType() - .name() - .substring(0, 1)); + ou.getObservationUnitPosition() + .getEntryType() + .name() + .substring(0, 1)); } assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); @@ -1518,8 +1574,8 @@ private String yearToSeasonDbId(String year, UUID programId) throws ApiException for (BrAPISeason season : seasons) { if (null == season.getSeasonName() || season.getSeasonName() - .isBlank() || season.getSeasonName() - .equals(year)) { + .isBlank() || season.getSeasonName() + .equals(year)) { return season.getSeasonDbId(); } } @@ -1530,17 +1586,17 @@ private String yearToSeasonDbId(String year, UUID programId) throws ApiException private Program createProgram(String name, String abbv, String key, String referenceSource, List germplasm, List traits) throws ApiException, DoesNotExistException, ValidatorException, BadRequestException { SpeciesEntity validSpecies = speciesDAO.findAll().get(0); SpeciesRequest speciesRequest = SpeciesRequest.builder() - .commonName(validSpecies.getCommonName()) - .id(validSpecies.getId()) - .build(); + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); ProgramRequest programRequest1 = ProgramRequest.builder() - .name(name) - .abbreviation(abbv) - .documentationUrl("localhost:8080") - .objective("To test things") - .species(speciesRequest) - .key(key) - .build(); + .name(name) + .abbreviation(abbv) + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key(key) + .build(); TestUtils.insertAndFetchTestProgram(gson, client, programRequest1); @@ -1609,6 +1665,33 @@ private JsonObject uploadAndVerifyFailure(Program program, File file, String exp JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); + assertEquals(1, rowErrors.size()); + JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); + assertEquals(1, fieldErrors.size()); + JsonObject error = fieldErrors.get(0).getAsJsonObject(); + assertEquals(expectedColumnError, error.get("field").getAsString()); + assertEquals(422, error.get("httpStatusCode").getAsInt()); + + return result; + } + + private JsonObject uploadAndVerifyWorkflowFailure(Program program, File file, String expectedColumnError, boolean commit, String workflowId) throws InterruptedException, IOException { + + //Flowable> call = importTestUtils.uploadDataFile(file, null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(file, null, true, client, program, mappingId, newExperimentWorkflowId); + //JsonObject result = JsonParser.parseString(upload).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); assertEquals(1, rowErrors.size()); JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index 12b79ac15..f5dd37f51 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -97,6 +97,38 @@ public Flowable> uploadDataFile(File file, Map> uploadWorkflowDataFile(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) { + + MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); + + // Upload file + String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", program.getId(), mappingId); + Flowable> call = client.exchange( + POST(uploadUrl, requestBody) + .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + String importId = result.get("importId").getAsString(); + + // Process data + String url = String.format("/programs/%s/import/mappings/%s/workflows/%s/data/%s/%s", program.getId(), mappingId, workflowId, importId, commit ? "commit" : "preview"); + Flowable> processCall = client.exchange( + PUT(url, userData) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + return processCall; + + } + public HttpResponse getUploadedFile(String importId, RxHttpClient client, Program program, String mappingId) throws InterruptedException { Flowable> call = client.exchange( GET(String.format("/programs/%s/import/mappings/%s/data/%s?mapping=true", program.getId(), mappingId, importId)) @@ -123,16 +155,16 @@ public Map setup(RxHttpClient client, Gson gson, DSLContext dsl, // Species Species validSpecies = speciesService.getAll().get(0); SpeciesRequest speciesRequest = SpeciesRequest.builder() - .commonName(validSpecies.getCommonName()) - .id(validSpecies.getId()) - .build(); + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); // Insert program ProgramRequest program = ProgramRequest.builder() - .name("Test Program") - .species(speciesRequest) - .key("TEST") - .build(); + .name("Test Program") + .species(speciesRequest) + .key("TEST") + .build(); Program validProgram = this.insertAndFetchTestProgram(program, client, gson); // Get import @@ -141,18 +173,18 @@ public Map setup(RxHttpClient client, Gson gson, DSLContext dsl, ); HttpResponse response = call.blockingFirst(); String mappingId = JsonParser.parseString(response.body()).getAsJsonObject() - .getAsJsonObject("result") - .getAsJsonArray("data") - .get(0).getAsJsonObject().get("id").getAsString(); + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); BiUserEntity testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), validProgram.getId()); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); return Map.of("program", validProgram, - "mappingId", mappingId, - "testUser", testUser, - "securityFp", securityFp); + "mappingId", mappingId, + "testUser", testUser, + "securityFp", securityFp); } @@ -170,6 +202,43 @@ public JsonObject uploadAndFetch(File file, Map userData, Boolea return result; } + public JsonObject uploadAndFetchWorkflow(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { + Flowable> call = uploadWorkflowDataFile(file, userData, commit, client, program, mappingId, workflowId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + return result; + } + + public JsonObject uploadAndFetchWorkflowNoStatusCheck(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { + Flowable> call = uploadWorkflowDataFile(file, userData, commit, client, program, mappingId, workflowId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + return result; + } + public List createTraits(int numToCreate) { List traits = new ArrayList<>(); for (int i = 0; i < numToCreate; i++) { From 651cffc5dc8fc31e27a0fed557cdf1267ea1e0ee Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:52:11 -0400 Subject: [PATCH 05/21] Removed unused migration --- .../V1.23.0__add_experiment_workflows.sql | 51 ------------------- 1 file changed, 51 deletions(-) delete mode 100644 src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql diff --git a/src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql deleted file mode 100644 index 2c9d4d547..000000000 --- a/src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql +++ /dev/null @@ -1,51 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - Table maps workflows to import mappings and provides required configuration options - - mapping_id - link to importer_mapping that this provides a workflow for - name - name that will be displayed on front end - bean - must match @Named("") annotation on Workflow class - position - for ordering records explicitly, wanted for front end default option and order - */ -CREATE TABLE importer_mapping_workflow -( - like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, - mapping_id UUID NOT NULL, - name TEXT NOT NULL, - bean TEXT NOT NULL, - position INTEGER NOT NULL -); - -ALTER TABLE importer_mapping_workflow - ADD FOREIGN KEY (mapping_id) REFERENCES importer_mapping (id); - -DO -$$ -DECLARE - exp_mapping_id UUID; -BEGIN - exp_mapping_id := (SELECT id FROM importer_mapping WHERE name = 'ExperimentsTemplateMap'); - -INSERT INTO public.importer_mapping_workflow (mapping_id, name, bean, position) -VALUES - (exp_mapping_id, 'Create new experiment', 'CreateNewExperimentWorkflow', 0), - (exp_mapping_id, 'Append experimental dataset', 'AppendOverwritePhenotypesWorkflow', 1), - (exp_mapping_id, 'Create new experimental environment', 'CreateNewEnvironmentWorkflow', 2); -END -$$; \ No newline at end of file From 39fb8cd9ccbddf39f8c759e6654eb7d84364a441 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:03:55 -0400 Subject: [PATCH 06/21] Removed append environment workflow --- .../ExperimentWorkflowNavigator.java | 3 +- .../CreateNewEnvironmentWorkflow.java | 54 ------------------- 2 files changed, 1 insertion(+), 56 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index a498a14b2..9b47c935d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -57,8 +57,7 @@ public List getWorkflows() throws Exception { public enum Workflow { NEW_OBSERVATION("new-experiment","Create new experiment"), - APPEND_OVERWRITE("append-dataset", "Append experimental dataset"), - APPEND_ENVIRONMENT("append-environment", "Create new experimental environment"); + APPEND_OVERWRITE("append-dataset", "Append experimental dataset"); private String id; private String name; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java deleted file mode 100644 index fdb2a9fcc..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; - -import lombok.Getter; -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; - -import javax.inject.Singleton; -import java.util.Optional; - -@Getter -@Singleton -public class CreateNewEnvironmentWorkflow implements ExperimentWorkflow { - private final ExperimentWorkflowNavigator.Workflow workflow; - - public CreateNewEnvironmentWorkflow(){ - this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_ENVIRONMENT; - } - - public Optional process(ImportServiceContext context) { - // Workflow processing the context - ImportWorkflow workflow = ImportWorkflow.builder() - .id(getWorkflow().getId()) - .name(getWorkflow().getName()) - .build(); - - // No-preview result - Optional result = Optional.of(ImportWorkflowResult.builder() - .workflow(workflow) - .importPreviewResponse(Optional.empty()) - .build()); - - // Skip this workflow unless appending a new environment - if (context != null && !this.workflow.isEqual(context.getWorkflow())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return result; - } - - // Start processing the import... - return result; - } - - @Override - public int getOrder() { - return 3; - } - -} From be8ea3844b378a1b4a9e71b9b5797ebe020c57ed Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:36:43 -0400 Subject: [PATCH 07/21] Fix error message --- .../create/workflow/CreateNewExperimentWorkflow.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index b3636a38e..29d7613f1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -101,9 +101,7 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio // Make sure the file does not contain obs unit ids before proceeding if (containsObsUnitIDs(context)) { - // TODO: get file name - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Error detected in file, " + - upload.getUploadFileName() + ". ObsUnitIDs are detected. Import cannot proceed"); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "ObsUnitIDs are detected"); } statusService.updateMessage(upload, "Checking existing experiment objects in brapi service and mapping data"); From d11ef3c63cc8c42c7860147be8a2767b317e96f3 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:42:13 -0400 Subject: [PATCH 08/21] Removed navigator stuff handled in domain import service --- .../experimentObservation/ExperimentImportService.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 013d55012..a9b6c62ac 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.model.imports.DomainImportService; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; @@ -27,14 +26,12 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; -import java.util.List; @Singleton @Slf4j public class ExperimentImportService extends DomainImportService { private final String IMPORT_TYPE_ID = "ExperimentImport"; - private final ExperimentWorkflowNavigator workflowNavigator; // TODO: delete processor fields once WorkflowNavigator is used @Inject @@ -43,7 +40,6 @@ public ExperimentImportService(Provider experimentProcessor ExperimentWorkflowNavigator workflowNavigator) { super(experimentProcessorProvider, processorManagerProvider, workflowNavigator); - this.workflowNavigator = workflowNavigator; } @Override @@ -56,9 +52,5 @@ public String getImportTypeId() { return IMPORT_TYPE_ID; } - @Override - public List getWorkflows() throws Exception{ - return workflowNavigator.getWorkflows(); - } } From eebb4938883656a5f6cb46672d28939cf584d662 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:48:16 -0400 Subject: [PATCH 09/21] Removed some development comments --- .../brapps/importer/model/workflow/ImportContext.java | 2 -- .../brapps/importer/model/workflow/ProcessedData.java | 3 --- 2 files changed, 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java index 53e3a2389..a7b6f7dc3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java @@ -40,8 +40,6 @@ public class ImportContext { private UUID workflowId; private ImportUpload upload; private List importRows; - // TODO: move this out potentially - //private Map mappedBrAPIImport; private Table data; private Program program; private User user; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java index 677f35469..ac3c5ed61 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -28,9 +28,6 @@ @ToString @NoArgsConstructor public class ProcessedData { - // TODO: remove this, already in ImportPreviewResponse - //private Map statistics; - // TODO private Map mappedBrAPIImport; private ImportPreviewResponse importPreviewResponse; } \ No newline at end of file From b5dc3e86ae91d9947c30974f227cb5c3eb2a3175 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 09:49:27 -0400 Subject: [PATCH 10/21] Reference enum for new-experiment --- .../brapps/importer/model/imports/DomainImportService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index d31f0a8d0..9c7a3f261 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.Workflow; @@ -71,7 +72,7 @@ public ImportPreviewResponse process(ImportServiceContext context) // TODO: return results from WorkflowNavigator once processing logic is in separate workflows // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); - if ("new-experiment".equals(context.getWorkflow())) { + if (ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION.getId().equals(context.getWorkflow())) { return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); } else { return processorManagerProvider.get().process(context.getBrAPIImports(), From 2bbf7f3158795c268dcbd2279ed90743f8223978 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:13:42 -0400 Subject: [PATCH 11/21] Change variable name to reflect string type Co-authored-by: mlm483 <128052931+mlm483@users.noreply.github.com> --- .../brapps/importer/model/imports/DomainImportService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 9c7a3f261..3d956743a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -67,8 +67,8 @@ public ImportPreviewResponse process(ImportServiceContext context) throws Exception { Optional.ofNullable(context.getWorkflow()) - .filter(workflow -> !workflow.isEmpty()) - .ifPresent(workflow -> log.info("Workflow: " + workflow)); + .filter(workflowName -> !workflowName.isEmpty()) + .ifPresent(workflowName -> log.info("Workflow: " + workflowName)); // TODO: return results from WorkflowNavigator once processing logic is in separate workflows // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); From ccc141472ccf40930141e2454df089cf55f9be0c Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:20:40 -0400 Subject: [PATCH 12/21] Clarified comment --- .../experiment/create/model/ProcessedPhenotypeData.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java index cd6842476..c81e265cd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java @@ -23,7 +23,8 @@ import java.util.List; import java.util.Map; -// TODO: move to common higher level location +// TODO: move to common higher level location, could be used by both append and create workflows so being located +// in the create namespace won't make sense if we decide to do that in the future. @Getter @Setter @Builder From aa06c6e64c5b651757bbc5b7631d23c1f113b781 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:25:39 -0400 Subject: [PATCH 13/21] Removed old comment --- .../brapps/importer/services/FileImportService.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index da5a68e10..0c2a9309d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -576,11 +576,6 @@ public List getWorkflowsForSystemMapping(UUID mappingId) throws .orElseThrow(() -> new DoesNotExistException("Cannot find mapping config associated with upload.")); BrAPIImportService importService = configManager.getImportServiceById(mappingConfig.getImportTypeId()) .orElseThrow(() -> new DoesNotExistException("Config with that id does not exist")); - // NOTE: - // this is creating a workflow navigator to call getWorkflows - // workflowNavigator.getWorkflows(); - // getWorkflows is creating the ExperimentWorkflow which is injecting the file import service and - // resulting in a circular dependency return importService.getWorkflows(); } From dd6f34862c92bd90758cb8bbad839dac8c484205 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:32:54 -0400 Subject: [PATCH 14/21] Updated doc comment --- .../create/workflow/CreateNewExperimentWorkflow.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 29d7613f1..f007af675 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -139,9 +139,12 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio } /** - * Retrieves the name of the workflow. This is used for logging display purposes. + * Process the import service context and returns an Optional ImportWorkflowResult. * - * @return the name of the workflow + * @param context The import service context to be processed. If null, then it skips processing but returns the result with no-preview. + * @return An Optional ImportWorkflowResult which contains the workflow and import preview response (if available). + * If the context is null, it returns the result with no-preview. + * @throws Exception If any error occurs during the processing. */ public Optional process(ImportServiceContext context) throws Exception { // Workflow processing the context From a6c35902c0bdbd3733a809f72d23a37a627bcfd0 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 15:44:41 -0400 Subject: [PATCH 15/21] Cleaned up ProcessedData code --- .../model/workflow/ProcessedData.java | 1 - .../workflow/CreateNewExperimentWorkflow.java | 3 ++- .../PopulateNewPendingImportObjectsStep.java | 21 ++++++++++--------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java index ac3c5ed61..fc29774f0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -29,5 +29,4 @@ @NoArgsConstructor public class ProcessedData { private Map mappedBrAPIImport; - private ImportPreviewResponse importPreviewResponse; } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index f007af675..d776e4917 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -98,6 +98,7 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio ImportUpload upload = context.getUpload(); boolean commit = context.isCommit(); List importRows = context.getImportRows(); + ProcessedData processedData = new ProcessedData(); // Make sure the file does not contain obs unit ids before proceeding if (containsObsUnitIDs(context)) { @@ -108,7 +109,7 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(context); ProcessContext processContext = populateExistingPendingImportObjectsStep.process(context, phenotypeData); - ProcessedData processedData = populateNewPendingImportObjectsStep.process(processContext, phenotypeData); + populateNewPendingImportObjectsStep.process(processContext, phenotypeData); ValidationErrors validationErrors = validatePendingImportObjectsStep.process(context, processContext.getPendingData(), phenotypeData, processedData); // short circuit if there were validation errors diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index ddb4193ab..f20c8a1a1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -91,18 +91,19 @@ public PopulateNewPendingImportObjectsStep(ExperimentSeasonService experimentSea this.gson = new JSON().getGson(); } - public ProcessedData process(ProcessContext context, ProcessedPhenotypeData phenotypeData) + /** + * TODO: in the future returning ProcessedData rather than modifying in-place would be preferrable. + * + * @param context (modified in-place) + * @param phenotypeData + * @return + * @throws MissingRequiredInfoException + * @throws UnprocessableEntityException + * @throws ApiException + */ + public void process(ProcessContext context, ProcessedPhenotypeData phenotypeData) throws MissingRequiredInfoException, UnprocessableEntityException, ApiException { - - Table data = context.getImportContext().getData(); - ImportUpload upload = context.getImportContext().getUpload(); - ImportContext importContext = context.getImportContext(); - populatePendingImportObjects(context, phenotypeData); - - - // TODO: implement - return new ProcessedData(); } From 1a6de32dc4422765faf51876e4d151898734f542 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 2 Jul 2024 10:18:49 -0400 Subject: [PATCH 16/21] Changed from workflow to workflowId --- .../importer/controllers/UploadController.java | 12 ++++++------ .../importer/model/imports/DomainImportService.java | 8 ++++---- .../importer/model/imports/ImportServiceContext.java | 2 +- .../brapps/importer/services/FileImportService.java | 4 ++-- .../workflow/AppendOverwritePhenotypesWorkflow.java | 2 +- .../create/workflow/CreateNewExperimentWorkflow.java | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java index 5ee6eb062..eb2e2ec50 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java @@ -158,15 +158,15 @@ public HttpResponse> previewData(@PathVariable UUID pro } } - @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflow}/data/{uploadId}/preview") + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/preview") @Produces(MediaType.APPLICATION_JSON) @AddMetadata @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) public HttpResponse> previewData(@PathVariable UUID programId, @PathVariable UUID mappingId, - @PathVariable String workflow, @PathVariable UUID uploadId) { + @PathVariable String workflowId, @PathVariable UUID uploadId) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflow, actingUser, null, false); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, null, false); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { @@ -184,16 +184,16 @@ public HttpResponse> previewData(@PathVariable UUID pro } } - @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflow}/data/{uploadId}/commit") + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/commit") @Produces(MediaType.APPLICATION_JSON) @AddMetadata @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) public HttpResponse> commitData(@PathVariable UUID programId, @PathVariable UUID mappingId, - @PathVariable String workflow, @PathVariable UUID uploadId, + @PathVariable String workflowId, @PathVariable UUID uploadId, @Body @Nullable Map userInput) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflow, actingUser, userInput, true); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, userInput, true); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 3d956743a..4da8efb21 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -66,13 +66,13 @@ public List getWorkflows() throws Exception{ public ImportPreviewResponse process(ImportServiceContext context) throws Exception { - Optional.ofNullable(context.getWorkflow()) - .filter(workflowName -> !workflowName.isEmpty()) - .ifPresent(workflowName -> log.info("Workflow: " + workflowName)); + Optional.ofNullable(context.getWorkflowId()) + .filter(workflowId -> !workflowId.isEmpty()) + .ifPresent(workflowId -> log.info("Workflow: " + workflowId)); // TODO: return results from WorkflowNavigator once processing logic is in separate workflows // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); - if (ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION.getId().equals(context.getWorkflow())) { + if (ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION.getId().equals(context.getWorkflowId())) { return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); } else { return processorManagerProvider.get().process(context.getBrAPIImports(), diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java index 4d052cd33..90e8915f9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -32,7 +32,7 @@ @AllArgsConstructor @NoArgsConstructor public class ImportServiceContext { - private String workflow; + private String workflowId; private List brAPIImports; private Table data; private Program program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 0c2a9309d..e24cfeef2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -420,14 +420,14 @@ public ImportUpload setDynamicColumns(ImportUpload newUpload, Table data, Import return newUpload; } - private void processFile(String workflow, List finalBrAPIImportList, Table data, Program program, + private void processFile(String workflowId, List finalBrAPIImportList, Table data, Program program, ImportUpload upload, User user, Boolean commit, BrAPIImportService importService, AuthenticatedUser actingUser) { // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { ImportServiceContext context = ImportServiceContext.builder() - .workflow(workflow) + .workflowId(workflowId) .brAPIImports(finalBrAPIImportList) .data(data) .program(program) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index 55c34e88a..ea5c388cb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -33,7 +33,7 @@ public Optional process(ImportServiceContext context) { .build()); // Skip this workflow unless appending or overwriting observation data - if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + if (context != null && !this.workflow.isEqual(context.getWorkflowId())) { return Optional.empty(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index d776e4917..ed95014a6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -163,7 +163,7 @@ public Optional process(ImportServiceContext context) thro .build()); // Skip this workflow unless creating a new experiment - if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + if (context != null && !this.workflow.isEqual(context.getWorkflowId())) { return Optional.empty(); } From a488612e506cd767fa7c5acdc69f57b6ad35228b Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:20:07 -0400 Subject: [PATCH 17/21] Change exception handling --- .../model/imports/DomainImportService.java | 10 ++- .../model/workflow/ImportWorkflowResult.java | 1 + .../importer/model/workflow/Workflow.java | 4 +- .../ExperimentWorkflowNavigator.java | 63 ++++++++++--------- .../workflow/CreateNewExperimentWorkflow.java | 27 ++++---- 5 files changed, 60 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 4da8efb21..6cfffe73c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -73,7 +73,15 @@ public ImportPreviewResponse process(ImportServiceContext context) // TODO: return results from WorkflowNavigator once processing logic is in separate workflows // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); if (ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION.getId().equals(context.getWorkflowId())) { - return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); + Optional result = workflowNavigator.process(context); + + // Throw any exceptions caught during workflow processing + if (result.flatMap(ImportWorkflowResult::getCaughtException).isPresent()) { + throw result.flatMap(ImportWorkflowResult::getCaughtException).get(); + } + + return result.flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); + } else { return processorManagerProvider.get().process(context.getBrAPIImports(), List.of(experimentProcessorProvider.get()), diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java index 9304c1ce2..f59b83b55 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java @@ -30,4 +30,5 @@ public class ImportWorkflowResult { private ImportWorkflow workflow; private Optional importPreviewResponse; + private Optional caughtException; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java index dec774e54..17f48ee57 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -9,8 +9,8 @@ @FunctionalInterface public interface Workflow extends Ordered { - Optional process(ImportServiceContext context) throws Exception; - default List getWorkflows() throws Exception { + Optional process(ImportServiceContext context); + default List getWorkflows() { // Default implementation for getWorkflows method return new ArrayList<>(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index 9b47c935d..6acd4aad5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -7,9 +7,9 @@ import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import javax.inject.Singleton; -import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; @Primary @Singleton @@ -20,39 +20,46 @@ public ExperimentWorkflowNavigator(List workflows) { this.workflows = workflows; } + /** + * Process the import service context by executing a series of workflows in order + * + * This method iterates over the list of workflows provided, executing each workflow's process method + * with the given import service context. It then filters out empty results and returns the first non-empty result. + * + * @param context The import service context containing the data to be processed + * @return An Optional containing the first non-empty ImportWorkflowResult from the executed workflows, or an empty Optional if no non-empty result is found + */ @Override - public Optional process(ImportServiceContext context) throws Exception { - - // NOTE: Couldn't throw exception from map lambda - for (ExperimentWorkflow workflow : workflows) { - Optional result = workflow.process(context); - if (result.isPresent()) { - return result; - } - } - - return Optional.empty(); + public Optional process(ImportServiceContext context) { + /** + * Have each workflow in order process the context, returning the first non-empty result + */ + return workflows.stream() + .map(workflow->workflow.process(context)) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst(); } - @Override - public List getWorkflows() throws Exception { - // Each workflow returns in the field workflow the metadata about the workflow that processed the import context. - // Loop over all workflows, processing a null context, to collect just the metadata - - // NOTE: Couldn't throw exception from map lambda - List workflowSummaryList = new ArrayList<>(); - - for (ExperimentWorkflow workflow : workflows) { - Optional result = workflow.process(null); - result.ifPresent(importWorkflowResult -> workflowSummaryList.add(importWorkflowResult.getWorkflow())); - } - - // The order field for each workflow is set to the order in the list + /** + * Retrieves a list of ImportWorkflow objects containing metadata about each workflow that processed the import context. + * + * @return List of ImportWorkflow objects with workflow metadata + */ + public List getWorkflows() { + List workflowSummaryList = workflows.stream() + .map(workflow -> workflow.process(null)) // Process each workflow with a null context + .filter(Optional::isPresent) // Filter out any workflows that do not return a result + .map(Optional::get) // Extract the result from Optional + .map(result -> result.getWorkflow()) // Retrieve the workflow metadata + .collect(Collectors.toList()); // Collect the workflow metadata into a list + + // Set the order field for each workflow based on its position in the list for (int i = 0; i < workflowSummaryList.size(); i++) { - workflowSummaryList.get(i).setOrder(i); + workflowSummaryList.get(i).setOrder(i); // Set the order for each workflow } - return workflowSummaryList; + return workflowSummaryList; // Return the list of workflow metadata } public enum Workflow { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index ed95014a6..d0a4ca975 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -145,9 +145,8 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio * @param context The import service context to be processed. If null, then it skips processing but returns the result with no-preview. * @return An Optional ImportWorkflowResult which contains the workflow and import preview response (if available). * If the context is null, it returns the result with no-preview. - * @throws Exception If any error occurs during the processing. */ - public Optional process(ImportServiceContext context) throws Exception { + public Optional process(ImportServiceContext context) { // Workflow processing the context ImportWorkflow workflow = ImportWorkflow.builder() .id(getWorkflow().getId()) @@ -155,12 +154,11 @@ public Optional process(ImportServiceContext context) thro .build(); // No-preview result - Optional result; - - result = Optional.of(ImportWorkflowResult.builder() + ImportWorkflowResult workflowResult = ImportWorkflowResult.builder() .workflow(workflow) .importPreviewResponse(Optional.empty()) - .build()); + .caughtException(Optional.empty()) + .build(); // Skip this workflow unless creating a new experiment if (context != null && !this.workflow.isEqual(context.getWorkflowId())) { @@ -169,21 +167,22 @@ public Optional process(ImportServiceContext context) thro // Skip processing if no context, but return no-preview result for this workflow if (context == null) { - return result; + return Optional.of(workflowResult); } // TODO: unify usage of single import context type throughout ImportContext importContext = ImportContext.from(context); // Start processing the import... - ImportPreviewResponse response = runWorkflow(importContext); - - result = Optional.of(ImportWorkflowResult.builder() - .workflow(workflow) - .importPreviewResponse(Optional.of(response)) - .build()); + ImportPreviewResponse response; + try { + response = runWorkflow(importContext); + workflowResult.setImportPreviewResponse(Optional.of(response)); + } catch(Exception e) { + workflowResult.setCaughtException(Optional.of(e)); + } - return result; + return Optional.of(workflowResult); } @Override From 94a5ff0b9b3c179197cdbea9d746e58b0085317b Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 16 Jul 2024 11:25:11 -0400 Subject: [PATCH 18/21] Removed throws Exception in method signature --- .../brapps/importer/controllers/ImportController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index 4211779d7..c9ea3ec39 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -214,7 +214,7 @@ public HttpResponse>> getSystemMappings(@Nu @Produces(MediaType.APPLICATION_JSON) @AddMetadata @Secured(SecurityRule.IS_ANONYMOUS) - public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) throws Exception { + public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { List workflows = null; try { From e5c7aafdf745960ab44f95f0bcd21b84b8790535 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 16 Jul 2024 11:40:02 -0400 Subject: [PATCH 19/21] Removed todo --- .../brapps/importer/ExperimentFileImportTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index ee34a4fff..50022707d 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1034,7 +1034,6 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { - a new experiment is created after the first experiment - verify the second experiment gets created successfully */ - //TODO: this one @Test @SneakyThrows public void importSecondExpAfterFirstExpWithObs() { @@ -1057,7 +1056,6 @@ public void importSecondExpAfterFirstExpWithObs() { newExpA.put(Columns.COLUMN, "1"); newExpA.put(traits.get(0).getObservationVariableName(), "1"); - //JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); JsonObject resultA = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsA = resultA.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); @@ -1086,7 +1084,6 @@ public void importSecondExpAfterFirstExpWithObs() { newExpB.put(Columns.COLUMN, "1"); newExpB.put(traits.get(0).getObservationVariableName(), "1"); - //JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); JsonObject resultB = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsB = resultB.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); From ba4c0a0ea76eb2e250504e278996223a4ea5e8f3 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 16 Jul 2024 14:24:01 -0400 Subject: [PATCH 20/21] Merged missing BI-2109 changes --- .../PopulateNewPendingImportObjectsStep.java | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index ff47e3b2d..0e2e465cf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -32,22 +32,18 @@ import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingImportObjectData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentSeasonService; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentValidateService; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.model.User; @@ -55,8 +51,9 @@ import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.Utilities; +import org.breedinginsight.model.DatasetMetadata; +import org.breedinginsight.model.DatasetLevel; import org.jooq.DSLContext; -import tech.tablesaw.api.Table; import org.breedinginsight.model.Trait; import tech.tablesaw.columns.Column; @@ -371,7 +368,22 @@ public void fetchOrCreateDatasetPIO(ImportContext importContext, program, trialPIO.getId().toString()); pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); - trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); + + JsonArray datasetsJson = trialPIO.getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); + // If datasets property does not yet exist, create it + String datasetName = StringUtils.isNotBlank(importRow.getSubObsUnit()) ? importRow.getSubObsUnit() : importRow.getExpUnit(); + List datasets = DatasetUtil.datasetsFromJson(datasetsJson); + DatasetMetadata dataset = DatasetMetadata.builder() + .name(datasetName) + .id(id) + .level(StringUtils.isNotBlank(importRow.getSubObsUnit()) ? DatasetLevel.SUB_OBS_UNIT : DatasetLevel.EXP_UNIT) + .build(); + + log.debug(dataset.getName()); + datasets.add(dataset); + datasetsJson = DatasetUtil.jsonArrayFromDatasets(datasets); + trialPIO.getBrAPIObject().getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, datasetsJson); + if (ImportObjectState.EXISTING == trialPIO.getState()) { trialPIO.setState(ImportObjectState.MUTATED); } From b284e1a5dd7be72ba52d1a22f763ddf863fabc9d Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:31:25 -0400 Subject: [PATCH 21/21] Update tests --- .../importer/ExperimentFileImportTest.java | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 50022707d..975c2b083 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1024,8 +1024,8 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { newObservation.put(Columns.COLUMN, "1"); newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "2"); - - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); + + uploadAndVerifyWorkflowFailureNonTabular(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); } /* @@ -1638,20 +1638,9 @@ private JsonObject uploadAndVerifyFailure(Program program, File file, String exp private JsonObject uploadAndVerifyWorkflowFailure(Program program, File file, String expectedColumnError, boolean commit, String workflowId) throws InterruptedException, IOException { - //Flowable> call = importTestUtils.uploadDataFile(file, null, true, client, program, mappingId); - //HttpResponse response = call.blockingFirst(); - //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - - //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - - //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(file, null, true, client, program, mappingId, newExperimentWorkflowId); - //JsonObject result = JsonParser.parseString(upload).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - - JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); assertEquals(1, rowErrors.size()); JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); @@ -1663,4 +1652,12 @@ private JsonObject uploadAndVerifyWorkflowFailure(Program program, File file, St return result; } + private JsonObject uploadAndVerifyWorkflowFailureNonTabular(Program program, File file, String expectedColumnError, boolean commit, String workflowId) throws InterruptedException, IOException { + + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(file, null, true, client, program, mappingId, newExperimentWorkflowId); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + return result; + } + }