From 35ebe96ed0303b0da1a47e2bc3c57411ab04cb97 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 15 May 2023 11:39:55 -0400 Subject: [PATCH 01/27] make integration tests for ExperimentController --- .../ExperimentControllerIntegrationTest.java | 528 ++++++++++++++++++ .../feature1465EmptyDatasetImport.csv | 3 + .../feature1465EmptyDatasetImport.xls | Bin 0 -> 36352 bytes .../feature1465EmptyDatasetImport.xlsx | Bin 0 -> 12775 bytes .../feature1465PopulatedDatasetImport.csv | 3 + .../feature1465PopulatedDatasetImport.xls | Bin 0 -> 36352 bytes .../feature1465PopulatedDatasetImport.xlsx | Bin 0 -> 12786 bytes .../germplasm_import.csv | 2 + .../ExperimentControllerIntegrationTest.sql | 37 ++ 9 files changed, 573 insertions(+) create mode 100644 src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java create mode 100644 src/test/resources/files/experiment_controller/feature1465EmptyDatasetImport.csv create mode 100644 src/test/resources/files/experiment_controller/feature1465EmptyDatasetImport.xls create mode 100644 src/test/resources/files/experiment_controller/feature1465EmptyDatasetImport.xlsx create mode 100644 src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.csv create mode 100644 src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.xls create mode 100644 src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.xlsx create mode 100644 src/test/resources/files/experiment_controller/germplasm_import.csv create mode 100644 src/test/resources/sql/ExperimentControllerIntegrationTest.sql diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java new file mode 100644 index 000000000..ea9c7b74d --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -0,0 +1,528 @@ +package org.breedinginsight.brapi.v2; + +import com.google.gson.*; +import io.kowalski.fannypack.FannyPack; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.client.exceptions.HttpClientResponseException; +import io.micronaut.http.netty.cookies.NettyCookie; +import io.micronaut.test.annotation.MicronautTest; +import io.reactivex.Flowable; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.TestUtils; +import org.breedinginsight.api.model.v1.request.ProgramRequest; +import org.breedinginsight.api.model.v1.request.SharedOntologyProgramRequest; +import org.breedinginsight.api.model.v1.request.SpeciesRequest; +import org.breedinginsight.api.v1.controller.TestTokenValidator; +import org.breedinginsight.dao.db.enums.DataType; +import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.SpeciesDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.*; +import org.jooq.DSLContext; +import org.junit.jupiter.api.*; + +import javax.inject.Inject; +import java.io.File; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static io.micronaut.http.HttpRequest.*; +import static org.junit.jupiter.api.Assertions.*; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class ExperimentControllerIntegrationTest extends BrAPITest { + + private FannyPack securityFp; + private FannyPack brapiFp; + private FannyPack brapiObservationFp; + private Program program; + private String germplasmImportId; + private final String GERMPLASM_LIST_NAME = "Program Germplasm List"; + private final String GERMPLASM_LIST_DESC = "Program Germplasm List"; + @Inject + private DSLContext dsl; + @Inject + private ProgramDAO programDAO; + @Inject + private UserDAO userDAO; + @Inject + private SpeciesDAO speciesDAO; + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + private Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .create(); + + @BeforeAll + void setup() throws Exception { + securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); + //brapiObservationFp = FannyPack.fill("src/test/resources/sql/brapi/BrAPIOntologyControllerIntegrationTest.sql"); + + // Test User + User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + // Species + super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + + // Test Program + ProgramRequest programRequest = ProgramRequest.builder() + .name("Test Program") + .species(speciesRequest) + .key("TEST") + .build(); + program = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId().toString()); + + // Add trait to program + addTrait(program); + + // Add germplasm to program + addGermplasm(program); + + // Add experiemnts to program + addExperiments(program); + } + + private void addExperiments(Program program) throws InterruptedException { + // Get experiment system import + Flowable> call = client.exchange( + GET("/import/mappings?importName=experimenttemplatemap").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + germplasmImportId = JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + // Insert program germplasm + File file = new File("src/test/resources/files/experiment_controller/germplasm_import.csv"); + Map germplasmListInfo = Map.ofEntries( + Map.entry("germplasmListName", GERMPLASM_LIST_NAME), + Map.entry("germplasmListDescription", GERMPLASM_LIST_DESC) + ); + TestUtils.uploadDataFile(client, program.getId(), germplasmImportId, germplasmListInfo, file); + } + + private void addGermplasm(Program program) throws InterruptedException { + // Get germplasm system import + Flowable> call = client.exchange( + GET("/import/mappings?importName=germplasmtemplatemap").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + germplasmImportId = JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + // Insert program germplasm + File file = new File("src/test/resources/files/experiment_controller/germplasm_import.csv"); + Map germplasmListInfo = Map.ofEntries( + Map.entry("germplasmListName", GERMPLASM_LIST_NAME), + Map.entry("germplasmListDescription", GERMPLASM_LIST_DESC) + ); + TestUtils.uploadDataFile(client, program.getId(), germplasmImportId, germplasmListInfo, file); + } + private void addTrait(Program program) { + + // Add a trait + Trait trait = new Trait(); + trait.setTraitDescription("trait 1 description"); + trait.setEntity("entity1"); + trait.setObservationVariableName("ObsVar1"); + trait.setProgramObservationLevel(ProgramObservationLevel.builder().name("plant").build()); + Scale scale1 = new Scale(); + scale1.setScaleName("Test Scale"); + scale1.setDataType(DataType.TEXT); + Method method1 = new Method(); + trait.setScale(scale1); + trait.setMethod(method1); + trait.setTraitClass("Pheno trait"); + trait.setAttribute("leaf length"); + trait.setMainAbbreviation("abbrev1"); + trait.setSynonyms(List.of("test1", "test2")); + trait.getMethod().setMethodClass("Estimation"); + trait.getMethod().setDescription("A method"); + + List traits = List.of(trait); + + // Call endpoint + TestUtils.insertTestTraits(gson, client, program, traits); + } + + @Test + @Order(1) + void getAllProgramsNoSharedPrograms() { + String url = String.format("/programs/%s/ontology/shared/programs", mainProgram.getId()); + Flowable> call = client.exchange( + GET(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + JsonArray data = result.getAsJsonArray("data"); + assertEquals(3, data.size(), "Wrong number of programs returned"); + + // Check all are not shared and are inactive + for (JsonElement element: data) { + JsonObject program = element.getAsJsonObject(); + + assertFalse(program.get("shared").getAsBoolean(), "Shared should have been false"); + assertNull(program.get("accepted"), "Accepted should have been false"); + assertNull(program.get("editable"), "Editable should have been false"); + } + } + + @Test + @Order(2) + void addSharedPrograms() { + + String url = String.format("/programs/%s/ontology/shared/programs", mainProgram.getId()); + List requests = new ArrayList<>(); + requests.add(new SharedOntologyProgramRequest(otherProgram.getId(), otherProgram.getName())); + requests.add(new SharedOntologyProgramRequest(thirdProgram.getId(), thirdProgram.getName())); + String json = gson.toJson(requests); + + Flowable> call = client.exchange( + POST(url, json) + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + JsonArray data = result.getAsJsonArray("data"); + assertEquals(2, data.size(), "Wrong number of programs returned"); + + // Check all are not shared and are inactive + for (JsonElement element: data) { + JsonObject program = element.getAsJsonObject(); + + assertTrue(program.get("shared").getAsBoolean(), "Shared should have been true"); + assertFalse(program.get("accepted").getAsBoolean(), "Accepted should have been false"); + assertTrue(program.get("editable").getAsBoolean(), "Editable should have been true"); + } + } + + @Test + @Order(3) + void subscribeOntologyProgramHasTraitsError() { + String url = String.format("/programs/%s/ontology/subscribe/%s", thirdProgram.getId(), mainProgram.getId()); + + Flowable> call = client.exchange( + PUT(url, "") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = call.blockingFirst(); + }); + + assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, e.getStatus()); + } + + @Test + @Order(3) + void subscribeOntologySuccess() { + String url = String.format("/programs/%s/ontology/subscribe/%s", otherProgram.getId(), mainProgram.getId()); + + Flowable> call = client.exchange( + PUT(url, "") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + } + + @Test + @Order(4) + void getSubscribedOntologyOptions() { + + String url = String.format("/programs/%s/ontology/subscribe", otherProgram.getId()); + Flowable> call = client.exchange( + GET(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + JsonArray data = result.getAsJsonArray("data"); + assertEquals(1, data.size(), "Wrong number of programs returned"); + + // Check all are not shared and are inactive + for (JsonElement element: data) { + JsonObject program = element.getAsJsonObject(); + + assertTrue(program.get("subscribed").getAsBoolean()); + assertTrue(program.get("editable").getAsBoolean()); + } + } + + @Test + @Order(4) + void getAllProgramsSharedPrograms() { + String url = String.format("/programs/%s/ontology/shared/programs", mainProgram.getId()); + Flowable> call = client.exchange( + GET(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + JsonArray data = result.getAsJsonArray("data"); + assertEquals(3, data.size(), "Wrong number of programs returned"); + + // Check all are not shared and are inactive + for (JsonElement element: data) { + JsonObject program = element.getAsJsonObject(); + + if (program.get("programId").getAsString().equals(fourthProgram.getId().toString())) { + assertFalse(program.get("shared").getAsBoolean()); + } else { + assertTrue(program.get("shared").getAsBoolean(), "Shared should have been true"); + assertTrue(program.get("editable").getAsBoolean(), "Editable should have been false"); + if (program.get("programId").getAsString().equals(otherProgram.getId().toString())) { + assertTrue(program.get("accepted").getAsBoolean()); + } else { + assertFalse(program.get("accepted").getAsBoolean()); + } + } + } + } + + @Test + @Order(4) + void getOnlySharedPrograms() { + String url = String.format("/programs/%s/ontology/shared/programs?shared=true", mainProgram.getId()); + Flowable> call = client.exchange( + GET(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + JsonArray data = result.getAsJsonArray("data"); + assertEquals(2, data.size(), "Wrong number of programs returned"); + + // Check all are not shared and are inactive + for (JsonElement element: data) { + JsonObject program = element.getAsJsonObject(); + + assertTrue(program.get("shared").getAsBoolean(), "Shared should have been true"); + } + } + + @Test + @Order(4) + void shareOntologySubscribedToOtherProgram() { + // Cannot share ontology if you are using a shared ontology + String url = String.format("/programs/%s/ontology/shared/programs", otherProgram.getId()); + List requests = new ArrayList<>(); + requests.add(new SharedOntologyProgramRequest(fourthProgram.getId(), fourthProgram.getName())); + String json = gson.toJson(requests); + + Flowable> call = client.exchange( + POST(url, json) + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = call.blockingFirst(); + }); + assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, e.getStatus()); + } + + @Test + @Order(4) + void revokeOntologyUneditable() { + // Ontology cannot be revoke if shared program has accepted and has observations + super.getBrapiDsl().execute(brapiObservationFp.get("AddObservations"), otherProgram.getId().toString()); + + String url = String.format("/programs/%s/ontology/shared/programs/%s", mainProgram.getId(), otherProgram.getId()); + Flowable> call = client.exchange( + DELETE(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = call.blockingFirst(); + }); + assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, e.getStatus()); + + super.getBrapiDsl().execute(brapiObservationFp.get("DeleteObservations")); + } + + @Test + @Order(4) + void unsubscribeOntologyUneditableError() { + super.getBrapiDsl().execute(brapiObservationFp.get("AddObservations"), otherProgram.getId().toString()); + + String url = String.format("/programs/%s/ontology/subscribe/%s", otherProgram.getId(), mainProgram.getId()); + Flowable> call = client.exchange( + DELETE(url, "").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = call.blockingFirst(); + }); + assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, e.getStatus()); + + super.getBrapiDsl().execute(brapiObservationFp.get("DeleteObservations")); + } + + @Test + @Order(5) + void unsubscribeOntologySuccess() { + + String url = String.format("/programs/%s/ontology/subscribe/%s", otherProgram.getId(), mainProgram.getId()); + Flowable> call = client.exchange( + DELETE(url, "").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + } + + @Test + @Order(5) + void revokeOntologySuccess() { + + String url = String.format("/programs/%s/ontology/shared/programs/%s", mainProgram.getId(), thirdProgram.getId()); + Flowable> call = client.exchange( + DELETE(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + } + + @Test + void shareSelfError() { + // Test that program cannot share with themselves + + String url = String.format("/programs/%s/ontology/shared/programs", mainProgram.getId()); + List requests = new ArrayList<>(); + requests.add(new SharedOntologyProgramRequest(mainProgram.getId(), mainProgram.getName())); + String json = gson.toJson(requests); + + Flowable> call = client.exchange( + POST(url, json) + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = call.blockingFirst(); + }); + assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, e.getStatus()); + } + + @Test + void revokeOntologyProgramNotExist() { + String url = String.format("/programs/%s/ontology/shared/programs/%s", UUID.randomUUID(), otherProgram.getId()); + Flowable> call = client.exchange( + DELETE(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = call.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + @Test + void revokeOntologySharedProgramNotExist() { + String url = String.format("/programs/%s/ontology/shared/programs/%s", mainProgram.getId(), UUID.randomUUID()); + Flowable> call = client.exchange( + DELETE(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = call.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + @Test + void revokeOntologySharedProgramNotShared() { + String url = String.format("/programs/%s/ontology/shared/programs/%s", mainProgram.getId(), fourthProgram.getId()); + Flowable> call = client.exchange( + DELETE(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = call.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + @Test + void shareOntologyProgramNotExist() { + + String url = String.format("/programs/%s/ontology/shared/programs", mainProgram.getId()); + List requests = new ArrayList<>(); + requests.add(new SharedOntologyProgramRequest(UUID.randomUUID(), "I don't exist")); + String json = gson.toJson(requests); + + Flowable> call = client.exchange( + POST(url, json) + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = call.blockingFirst(); + }); + assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, e.getStatus()); + } + + @Test + void shareOntologySharedProgramNotExist() { + + String url = String.format("/programs/%s/ontology/shared/programs", UUID.randomUUID()); + List requests = new ArrayList<>(); + requests.add(new SharedOntologyProgramRequest(otherProgram.getId(), otherProgram.getName())); + String json = gson.toJson(requests); + + Flowable> call = client.exchange( + POST(url, json) + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = call.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } +} diff --git a/src/test/resources/files/experiment_controller/feature1465EmptyDatasetImport.csv b/src/test/resources/files/experiment_controller/feature1465EmptyDatasetImport.csv new file mode 100644 index 000000000..1ed2eb60c --- /dev/null +++ b/src/test/resources/files/experiment_controller/feature1465EmptyDatasetImport.csv @@ -0,0 +1,3 @@ +Germplasm Name,Germplasm GID,Test (T) or Check (C ),Exp Title,Exp Description,Exp Unit,Exp Type,Env,Env Location,Env Year,Exp Unit ID,Exp Replicate #,Exp Block #,Row,Column,Treatment Factors,ObsUnitID,ObsVar1 +,1,,exp,,plant,trial,env1,here,2022,1,1,2,,,,, +,1,,exp,,plant,trial,env2,here,2022,1,1,2,,,,, \ No newline at end of file diff --git a/src/test/resources/files/experiment_controller/feature1465EmptyDatasetImport.xls b/src/test/resources/files/experiment_controller/feature1465EmptyDatasetImport.xls new file mode 100644 index 0000000000000000000000000000000000000000..c94875321d543122970bc8f6bfb7c2102b1dadec GIT binary patch literal 36352 zcmeHQ349b)(tne45(o)bIE4lRB#;9VF1d0OjvNp!5im@W4jD*h!ptOug_tY^1w}+a zIaNTB0CEd*-=Z99Kq8AQ2m%Vrp|UEniuwN4Jw4OYGZ})ruHX0j()smFzv}m@>Q&XN zs@LyLr~K2pmsf6SdW9*DzRZ>VR_Vn&WN;O(a=`1dCIWBugwOiI=O&@%Ug5J3tH(-RkGj3BpA+2!tr*Uy zT%v?}FA1Nmh0i+pRI&GkHeU#k&a4Id4)xV+9yJ4TPgN|_upFjk>CAx7|7H!BNAR%% zELIl(`y!_w(dCgfdOxCuibXo3+-mX;iIrqjwQlc$Fpo^GTW>Q*5q7AjCot| zDeb<6TXW_P4YAe%t=j~*Y11h!sMnZ4sW2v}g)>z?h(3W811E3PfwO55$%0u=TVi4lZ0QNC$w)>vI1$m{(6+f?4QGb0E@T%}vRewtrc>ZX9b_sNm%828|qL=+uz_(Wcf4>U&wkqIaf9>gt z{CL^%vq8Y)Ex{Mot-Gk!qFT&N)JrW3JS*!!%L6T$o5+tRe2Cx&enti1knA(43V3i8 z@D2{)wSiN>|AQGl6Y={A9b=$Ef8++TwGS>YUGC5`Ri3c7Wr z`WE?!_ES%SpJn54*;K+)UxHisRM_Y8x^?|)`9sd)xN9KQ2Yyk`(zx()C?Dj)4&{Rm zitw{Y8ChS891t}f)(2ShGn|+t zZX<_4fgk(&fIGwwxI=l0cm(^gp)0R1m#3T`Ib2rX29K=11Gqh1)Q@7~f%?Mv*#%71_Y)rVH~y{@{u$n(k2Yg|wwZ9b|eg_6?}YV>J*b1dl<17&sYC!Sju^Ld|d8`H$F&?V{g_6f=Kr!aA8c^_Y z+8|mn9w;igRy`ivgn`0mk;#cO2#zaKE0IvBss<7=!>W-`T&o5WGR>-yP!OyJ5;E7S zkx*o;1`;yis*zB*tOgP?>#C7ZEO;c6Q;E*CLxH#&NStd2=}L8wIM)t>U3HK+*A5b% z>L78hoo3ZT;#@nJ_dlAG@CfJH!E=G?AaSmp7S%)ITsz3zs>6wM?QqpY;#@nes)xk6 zcJL@g)*TdVBfaR^7;~Ue-$yuNbcRcXF=j=MqFH6T*;XawF8)! z3 zr+I-gPY!3Dg0)XeI-!~v_^1J9r?w3w8$0%`+{To-4IW#{ z+z6@EhAP<(0iC-MJga>)lFCTuNFu6(L>X?BdhG%vRztAsj=ec|MFZ;?i;OIAEX2xF zOYF?~KR3pdO6g`1g~Nd_4d}`em4zvL^XP7aWckP{%%k$wav2ikXzyVW56RMQJ}KE) zW#uUg63GQ^-MZCow98@AXeStO3x_n?5S9>RmS-m^`v$#i8YoRwRiqxS2nW5TYIT2wJCnRiFW(y->OE79rhdsxUfLe#S17RyUjzWX~ zD#yx!6b%NfVbhLdFlF9_MtZkw;t)TMI_S1@=**(OgWZ{(m5qp zhk^fO>{mw86fp3+)bdA0Qd%N?q+0$+sSK%}RQ}jVs#I0#?`7(b96>|>+p1EIpkdWv z;Ex=_!>hx?@ z(g)kbIUfJ11nsXCnd`4?gS8ojZ=n)K`+}Vo#=n6EgJl z#4I#VKCANHDaT|QD9LoSCll&~3_Wu(lTq_o7jNBnOs1idOgDQn;ZDeiPf-M+dGT3a zefx!DGX6xSi*rkB2{K)(VQHrl-#hk=V=|4DWE@yp8z*GsmUb%f=aU;8lWD9Z$A9GToe88V54ns$ps25p&m8p_?a^WE@ypfD*R!t z+|t4$mL2}uF_~6MG7c=QixV<(OAC)!TRztjncCQAjfFBx+>BPl%_k~7tBjS5#Tkx@ zNGVX_FvrT;tc?{ZEJ}&Al>({UiWs|7)Y&o804q}1j}mEH1yZ@qFh*-kLR43B+SZB` z)}lljq(CaS5OnzIr5}DhScu<}k?wVZ{`FvF3S>SE z>=eQtDA$*?Ps%o%mlyhm6~P3(9){Y>7O$#lr5j6t(DVvZrNvAM7l=u?>fq-Wcn;H_ z*JNsS<2lR~6pXt1L9+e95ZS&KjE)C`;@T&n^TdFO8#X~iN2cfr&O}GJz@-i*88YPX zeM;nY9UvF|UlqtJBUkObf{qaK2sv{2Kqc~e4v>p}unOd2YTzOu?;=MIU#M^huJs)t zCx0kv%tkB=T)bIVD3T4(<>kW^@@-Y-(g6J#0LkXT7D?&W;ZKXFlG12}_1_%+en^nu zmlCzQiF~Gc{0~wv(-?k-XuKrrP5gXyx-AYDZ;ThhVFFVO{gXT|EzC+TjTTj{j#ey5E=?>+E=?>g zaH++f#8WMnl5Y|;K3PGuAyB6XGO*XV+~t#pw9z>Mtel7?i4(CTaUzz;VfHL>hwK7u zS#WoUA_k++2Ft{sL`J6Ci%fNx)|3Njb+&Eg>I?M@gLRmoIFgsWwW|n4ZvlngL>8cU z_Tbt?KtTc*weKbvmCf@7lDp(b#NJ9JGO@a>4;&90sWIrtWr^-J(w0$|TJY2(LBRk+ zmb{iDw@N`=knmjfQ*5s6A;~=St}gzD0ZF>h+o522ig1;LzLPV|huwukidt46Sdg`b z{MUxv*f`-gwKLJH!%vZvJy>ddbX-#WO#Ff{M?(-_iWs~>n~Uc)T%5+FS*mW#mde-> zEt*kmVQ=9_%5TZ;5rR3ea25Uzsnj?zE&qoL`<2!6r9nYk5ioy>)#v2r>Ns-syaDAA z8iQ7op2Ks)rsVNJm&#Ad*Bbatc!)UOm|@W7QRaf3#tFP37vap9%Oz`a`T9b1GxMs+ z;o^9cMw^3Y@5%aiU^m%3r$a4HpT(JQJ>JqbohNPa1)7|EjS222L!;x;c}}a#$jQ${ zS*sHon0TUP;)A*Le3PvCWNl6kmz*+;+T?0X8QExJ(hE&8A+L0ek>}EaKrU&3O{W>tlt?eB1OMM{x`GU;!xXuKJ=|~_l z5#o>&K;2xTJ_9aNufuYybZHqOx&p01ucILQI}INANAVg1qBwnV8G3_(H|FVef@?L0 zBmUo_g6Jy^b8bM~QB2M9F;YU{_cMiHt;xyD*685M4B8B3)Ug?Lhk zO>z&xT#j}kFZc)!h|_rkf&*d=UTwlt;F5?Lkst6Fv%f=hW?z`f=jCXj9iD4b&e%`^ zA6$EmnXIiJ1lVA%4{ajE^h227MVE#&g2&97+!_T=WD?e{i$OEO(wMgfwj7dRCZ{oS zc^ZR>$Vn8Cs2KG5d0gk<@L&o@5x=98X3fQr2jkittfGU0gEQK3O~Fp^_Mx1B0Bnk7 zGxx?kK0})|h0CQZzyu56I*Yy%`9w6rwzdC8E?ZN;a~czu!)p-bEcG5cM8kik>L+8y z++T0d{zb1tI?n}83DF>53E~i;v^319C*g5iECdSv!Iv>l0~^8QN6a_Td<1VWLG5T< zXl|UL*BdZo$PyekM})H^0a=!qDGGrshzL`LN$C(TB+Xfx4CFQTf)&j{lLyO#Obg68 zFXEnH^vEtu+8pg)kOe{OlYz_tn;1`dxG@xuE>hBs6mSQ`C9CL~Hx!7p35ek-=7g4< z?NM2wRY?$rUWQH>tD)S;Yz(+OcoaA&j6tYXOwerFN!c0=QDg_OhfL(PqzEDV6Y{z| zAkri3!bJ#quaqz&akKQuD5cMkjR6WKWl&`!59df}yvPV*Gz2Yc1%**!FXibZ0=<;4 ze8GbNGcUSEzL2a1@!d?p7>b7kgNaM7GsG-hQJN4;(}-YrEddYFG55+m zO#m&c*o>iDWM*P#92_iGpQ9f?#iDg-f)L83NVlNyTq7ss;YbTJ`7Est@-+9+oa~!( zZ<;zdo>ovUO1x$ZQ!C~4#KHz8o(xY3a&Ju;d zQ8+@9l0xF*hGl2>%+2j-G|DPm|{h>AwIFu?nX102>8FD2`EP9dbU_!ja=JmkTAhKFgvUNODx3=f8} z126(#XYnh%5{Z?XoyVSVBn<2uY~gOIVr*;_WAjk`WGr6b2In;p^ME4InlON|*HLlR zAjT>%7tT=o<3vOAfoY6zxUkpMBWu4VyW=y*g$*9$>ec|$eFC>)H!t}f!1D&y9Ft|t zqS@$Jh#x~ejdEz?*wDh8PVo7N`)~BRYS`D(bNi9S%hwFtbT6i3z^!HpV^@FKJ>~p{ z5wri=v~h3EYTv~_?B2gOu>12R17dyzz-!`^=qZXgTlgGym%0ahaif*IjpPe&VSOb1fIY(d2Ts z#Ls^8Pd(Rt+{Ty7j#q9w(&^M2DTj(T1z(SM({*2%sJ)iHvMlWC(0%jD_;qJnlyBep z;PZ#*ihV~XPI=9L20HixPWtUA#z-z7Ci_gnS^{R=WFfUrC3zp={GzTDt(-jXeD{n! zf$`~EPj1}v6N_EmH2T98XWqPU?)udk-9P!VLB+B`KBIz{d-}Y1m|4&u;JG?3mYGQ@1^=*4*1A8q^M2ByqO!eU*taD-m#+9x4{>m*%dIM&x;8o zZ+vz?=&Sf!9ou=k^l;34@|oPzpJK#pZpj#p1{vF87djK$w?_@qH|y@VztN52r*7}t zl6k9clUH1CHEh2nr%h_=wf+OnhKAfVM5h**I?f*&=I;G=e3<)RZmkS#x%!xQ-!4gWA2U1IC?th#!EMjWvv{1ewN?tqOg8-%lb5b`MBTIqSa@< z92KY^x^vFz!L?#fOdZ&DM?jZ?E(gkJ;iY$5ojYA@+_mdq_SUucuN~qpyf>`Fq7HkOJnOT<(D2;G9Ubos zd?&@VJWt&|Y1iL+u3z@?ndTEq&TOi4al?d(>$9);JRIF|`Q(%T$hguqup;g0Lz8Qi&m3CRYh9 z%fEYi*V(nYPhR~daMp9dyU+S(_@B(*;=eNEST8m>>V<&WeV!cjU1OhCFD#oiU)BCn zU#}STUbhbt4;nPv%7bza75^A?Q~$+V+0EL-KK;E*yL5iQ_un6#w;=BGhn_1cx^wz3 z_dYrGw}0J!pdTOh%}Q_W$zmW#6^W*Z=07!e8#JpS&q;-^07>Ba8oXvwZ)_(+5lM4=9~Exg=#@Lhy9g zobdtgc4p{-Endgwe%7b$(XA5~)qb*mPVl}pM<;S$bs6wx?RTS2%>1dp+p|k{si&_G zxnO#;(5K(nHM%SHn|ZzOzN*f>qQsDhOR+;|w_bFj?+azx?}`Fq8y&wi$2V!r=>FfC zN`n6RMVsK#L0`{1GR0?2WNGcI1ydJ&(z@C4wo7)J?zTCWnE7JT(pE3jYqTTbtMa3l z8r4@1x%Ncoo6{#RDSEza;&-kuoZJ12*P++;e466+a;sfS7v5t-4kYe6vgm%NZC}0L z{jH4Bv7@pUC5$Mzc>QRTCtlKyoB31UV+~$kRrlDXtH0Fo@Oka2Zl4Z_RJU4xTQhQ8 zpQwU&*WGGdD|^APv~5dU-=DoXX2CkBoZWw#?w{Mv z{OLmr$kh=!tKS{8qvx&E6PHqFY^!s>WzNK>vzxZmlm)DR^42@+cio)u zYh}^MZFis9f0O?_zj^n4bA0<8OA3p4^W8BiVGG_5D?eYRP3U?3bi|w4cRm`Q7G~q0zz8`=+ExW2g4c-`(YG z=w%E*NzB_xw>`##_OTmZv+j!@sNAZ(6r)0%k)XZ(`NkL zpj)wJNl^Cf*FS%FVnmaetXcDSM?N>Z^R$=h&3aw51)FADHm{`WdeVEStV(>XjR3o0SI@mvTo>RICcWJWE|TX>j6(CtH8l zE_hJplwU&{zBA>Eu4(Z%8xK5JHz}serogQ5lq)@79dG(!c$Vwc3D0JAD$~rL(yF=J zx`RvmZC|Gy(KWTs#PxR{#GEknJ9BE{JFgBOR$&_7dE)c&O`h6%z2b$vhbCy!KhkE+ zOc{FLaAHG<;bQ8W^M-qMD$I&ZsMr@3-Kp&QvQcLnT&?KxuHmO?6&s&;(e;hkJFVjn zfAp_cXMWLk58w6ht|x+@S$huxJA3#{;pveR2HhI@_1B|wyY+bP+Ue<`lLxLY-&lG+ zG@;z@M^noD)jlD+|A_|~4&t}Z>3MJNz@27@?(G#JP)u5cpeT#t<|HP``*5cQrT3># zR_pbj?e=@6-Njku8_v3xUhC@9Eiz!p?K?gjgVmwWYxXbE?pgHu@%kCzmpW&!YpU*f z^VLa959RK>JvS=VKY8}~UYVQ6O`ogpey8F1R|W+pwF^iM4DGzwP^bOC1QQC!4aR;pZ!oUq(tvIY2WFh?Z=)D1A4iLV z^v2)X_{_u#Bwg3VCoM9n*{^HAKa-R`I5JIucM#wmh0o~%q(qq12q;-wtTHnerTQ!G z9KwW$(fg6v454zjLEwtYgug>QYuN>N)-|H{En@yz_=w~gvVP8mvcVEkW$Xvy0kzc@pO(}Fx??r8>U1-6UuCEb7yt_G)5sWnG&2C`A zX@yY*Z}zsJ{}OXOs^HDagtkoOE3_q9d$VDL0<=XKX25nZLZobqsj(f51nA9#cLhP> zB0gBG9X_%yB|e&#d9z=Ien`B~v|0(nf~y^jWaZ6%Hp^v%&^MB!H{;B55n&`jH-!xF zWB{;6)Ex`Y^#qtLFEq_|V=G8JmU?8LZYU9qczy_qbo3WK#gYmosvwqJusw&^6icpb zAZ&uM{dljofQdy}{=9@b)G{74#1>>>@fgoi$%Vzic}gW$l&BkojV#@YD+-oI6p3=Q zFgC=J3MB~}S!*=yR5I6eMTvSt*jyZ9BhF)l=`>-ZtvCHF*j&-Z${SgJj4)!TCiNzo zwh;)FXtRNp3pX1s+|Vu>dmY4@#AyP&i&T=+!%s;-L6U{LK##Od^avUaenl+SRgYS4(I| z&l9L7jml_x*|fPN=aV9Ul^*%LXuSDdtVtS=rk9us?R;$LQT(vB^AXx{=9;Acet765 zwyT47Ry(6N&G)nDy$;&-qZeEYR$B*_HeB{6jsdcte%MDXV)M1ZMv=>k%~!w{V8-U_ z5F6Q83=1=3BX4Zw%@5eDW55sCV(>F5kuF(dENf>esZf$eB1O(v%Sfz?cCkV(NU|r} zjKSt&5etp;7<$#VAW<5pG1!(Xm55$PbBVlH3|4=|d-a8OA(oPS?9>_uc7FVYaP94>4HWLpc= zp1|mKC1VpJAwaIKOVBA+YrMJ<8!v0*!c?>^-ZhSV{G&4!Lilz%G8O)RbEcAnk>bjc z$EPzBr*ud#%4JlzBjz#?3=S!Wk-k;PV}4&QQ{g>Gz$oQ172b)KGDQdZeIb{T-FJfq z<@ep;J>~7y@QM}cL{g@Z3wgo5*LhgW-q2%-tXvCTM&90s;)UlhOZz&A3hUI?vM(&B z1NfKl`yunJ@J_YxkMX9G_>*!gvI-`QC@Ht9XU;!LR7sZHsve6X@eA>t>JbGwj7B9L z$&i3Jx=4R1mMBX*8wXpWByCYm!KQ1erW@`_5sh;76ib|x&@l4x`4%jB2uwV}uMyswZ2jg$OL9!~O7({Pe^nT3*I8y(e=*QZR>;Xg3~T#1{qNNe3BO@lgCQQF`EacPMIk_W{^4~@6M8bXi z`kD2`8HqCrC(W{BaK_@K_78E=_y3n86>q1}<4rmoc5KtTfarakk8}7lcVUks$&2iW zQ`4vT{03kBgo62_ylwl3v`2#EE5Sz!Z^xO4ciIpi6D4r9zp<^nhMfj;HUsd05kvqAOfIZuCJMZ005W}0Kj_yG^n<)ovpK}t+T$0 zhrOwjF1@>rHDNwDC}l1H^zHfo9sk8&U?6eIwvQ2R@GRj5LA6>LXOmI(eNg`kE_-1e z)G)R%$t-C&d|*osjzM}c!Y6^1B1N3XM_&hH#G#`>Gn!C~P8!jk{V+J4ohFB$N}cjK zKe^+Gs7jKq5v1xsP`78U7wp?%K4O`3(~8QXo)!(12a6eg2jf^zyI-Q*TI5I(RE7Nj z#kqfNYr2EyUy!K<&xFm_CqqYPDi3_=BnvZ8G(lYL{aS%a4GyFhm7!LbdvzRfLx3#Ti7ZpUYdjUo}X%9W4qI6J}G4(^KikbhOziiHSfw!}wwthaknDk@l{!CK+Bm zw--s!vJ#}UPE;#G469|2Z!;%AvAD?3F!sh<@=hfBGrt4U26ya%hc8w9cw8Wf4ee%; zpC1bEu1*E?Llm*w{udBmD}t@fNA55_LN~`xa}~9A3vAqowBT`Y-t54|qF2`+RQ9gR zd08yj&LJ%_PBvF>;rIFq22l97;A~Q5B)NL?IGMNDgMAB5eMeJkCkFao=l=!ee{m%K z<<%?WW##)B;eyX3or6y4hl_j6gxOo zTAC+4vOyv?cOg=TG)1qD4fDPTHyE8a&0lL!R&(9xt{P-PNcp5HxVD8o_b_29!*4OU z_yC?ilvCz-HVtjq$;f=U+I!fV==vU8RoQ~mveqcuk&DtHoK_XRFZ20i@Ma01X0Bis> zh`TkzKjOsA&Jk#2X9xUM!~PXBAa8~24duW4=!~DR8eoL`bQaVdIMd}2|DrI;R;O%Xl^Ty@eC={9iXCzf#|<`!UI>;m!x?2rl{R}qUg1DR6;HC&9( z;X@q_>jeLequT2s?B98g1_HJ~*z)w(?je5#u1GZ*D1U~bp^Y}5AsrvvvAz|? zver@|5qeycI%+hmny*9;?WrpyCRDN&k?8qGM4pE4e7aw*h8EO@_ll6F&;cscB5hw? zKGsP&sVTV-S*d*hI^eV?OPZ3kDao${F&crb*5^_%NwQ?rRu`Hz1u<+fv1mf+=6ixB zEsNj+T!f{?mnc!z&0+u|+z@gG&v3l;yGYdZ`?E%p=dQ{e5m{ zN0-9cg7g4qp6&`)&vo3a=?X)o(=uu*4LFLHxf9*)J<@RqkK`Bv62Qq9n^u!)C;^yI zG_iC@=d#nX$tp2)qCR~z@R$Zasl{stdBdPZu6~tb&JD)Qx{1olpm}|n6iV*_7Eh^t zeFJP+S$alNq zUyL*EL-P0^ZHYfH%z>F}UyGqNG_8p1nHu5b*MWWRyyD(!uU&IBkKvi~2>mu=?;QI) za>oS~66R<5iFHAt3PLxaaO^0q?bHXXVa-|#W53X3J;HYl zZc89}o5x0Fs+Fg+jlJ{43Eb^hC9T3~ShK_Pp#cYRvB0JWwF6s7>`SCBx!{5@7R-)j zb~5Ev662w48Sg3l!MV@p@dNB5_Eu76+1I%rLMsq&UZLGuuomDOgAc-O2~r)}Q}91k zWEtetxy>W&*1?$iJ-(urAevLJlSh;c9kh+ktdT}fF2?Ns+&>e&tYY`~!)6bj6~MWH z=zAtOy!Lwwng2}3eWJVElHdS<5jp?>|Bs+@a`vz`b^4WtziDX46tJQC*37?te44a} zVMGe5WJ?HZoNX>=RW7SRNFC5JCb2h5sCwA&iX<{5PmmE}K51ltVY_kbcDmrai{z_v zGUJJ>&kqGoQ>MQ8zEC|%d;xJAh9%V@ z?gKtqK-g@of%Wu)A`9-GNKI>htIZLK;^UU0J_X__+duBk>Jhk+N#g0O%yDa(+ zm1Y8~*!P3kY><{*pW7)!q>gH2t5Gz}K<2LwQQ4R=gomV5_i*dj6F;|$0R;9Oge%GC zhZ74CD0aAu{64~Dn6j$0w5RRaF|kBt@9-aY*~0UuVfq*&JN>sknmYC-E~|cm)+8@0 z3b(zYjzVGHshcRg2U(pGO%qbGLL3BKeO2B@CQ19mL3vkmX~{Z}eXtZJy2<^p z5G*#R{jg%$Hcf9Twog=W4!)Eu3)b)|s(0O`^2r(qq>_EAgF-{{Gg;M8zIV;S@=ls^ zyoadA`xG=9eL|9%xT`pngkS1@ly7;AP>FNZ_u=cZT5nq=E+xf;Ef@yGs?4GCf=Yz2 z1(?FE3!Z*%*~9#MV2oyj3#9x`axlgGZ5G2#ong|}J&JY=KTG>2=S+E|xB|zuf)A=> zdZ{O@e@NsR89n|;`@)iz`CiBfa>x^jNpI7uEH~b}7{txK3I*lmZEBFBZmAdjx|d{2 zgrbTB<#8OQMpUwAkTf|mP}LVj3MSH6w$K+}zTK{DRIi6i@IG_021J9+3Q~O+7@i(% z{BmY5i;zx;3oqL>zP_s{1D_P)1(yEnAD8q^PaAaWyhT0IP2G?ww~^04pDx}0Z33Yw z{ChCRuRmoLcZ+Z0K^NmE*Y=ht%)LUGRM@G+ddhF`jsTPDiJ$52le7^HijUTKN7ZZ~ z%d{M>;ej*|OGTvpG~@&=491OKFF}RQNr5LBsy#ClM8HtYlkA@Iqn`JaSm%l_O#wj8 zC!C7-!TqVn5gXR7oHjEHd7(KRq>ad@k@>5xjuB_B2N>eUvHxw(`HX?oE%#Q$2MK?X z7=ILUXA4ssQ-(jzOusVEvF1bs4m)ZO#sfc`liSmW?HIE4%_-}IHBytTMBJw4V?{L< z=J-yW_n>5)*GiOyMJYmdyos{{VDMd6F)*Z!2Lv+|6H6pjJER%bQd+L?5#ldhB_+4n zzMemwFJ`+^J;hUD`{I-JyA>{ZGb5K$N#~ONfE<*p-trQOk;tYnw)e2KPtvP7zK+N? zBhW)jVD}={dC3HloU7F6U!a5IT6@hn{5QxWha~z)a?pox@dDRV(P@g5sxV37@ALwU z6uf+xiAav&lcfB-$~xu0GsOwPM^bL6m)DWy)ou6J3xx*lE1lvIty~&rGjZc@dClHZ zqM-O%+><@;VYD!y&!~}$>_BP?+vdJ9x5*fIYs@%Z9(=Z%C4>&t@+#5?E7cjgK6=*b zX`U?sv0x2IDsEolI!dyQHH24qSjg*8KT{n;U!T_3ZD_+Qj|p1LRq?A0CV*?-6osu3 zq~g=*56LnsF+W<&Mb+2tUz9%4Q)syYDu_rrwg%G&HCef}t)Q9o?MJ^~hYJ+!il>@V zjD@c_R#KMHAZFjuiacfJ1AGKkB9+`4`2r_D4PDTjjGo**t?`r?PU%$Ss ziCWqgicDGgbRkCn_4yfNUBBn${P2SPV4mWlZ=mPtek`Tu^@8iVcIz7&L$}Z6oDlT5?IZ;^MdG&YWGx7*Se}u;i5dN$IMxlXt0pxwx($leFP(5wJ!#?kRSRKSS<49LbH?Ga{^s)1E`)rxDgr99;9bwaPz*Bmu zO2F+NORoBs(sER-=`MUYN?0++=UL^RI^jsXiVoq>okd>eEt%S|t5c+!z|c7|VMhZp zCHh8TqtLNFQ^HRxmlNwYHT|W)1zv9x62rH3F#3j;?F^1t&?_D*af&c~!0V1M`N8by z_gS30nt@crGo0$@gE8$V>0PdnYiz-ucW~PXkNCO+Z1K{L`?GBkm@IL;u1ir0JWeYz z@xueYKGuJAN3HBE%EaC7}EOIqo`RCbOW#Pn+6bJCbLWEtCQn^7k=NWwdos-zj7RK3nL z05Od(2Y#h;Oo65`y}d}ZsWrEA2yjQI3`5>1yvG(%o!{nT66p2pX%VaPy#o%f_09Lw7Zi}0r9W&%kO7{Fr~4E+uc zV=W*Q?L|iRl6i;rW_--IhkPtE27mtZt8I-1b zBK{39w*<66PmAntz4q~yPG3D#o%9y&ULVWN=Jfb#OecY9KeYz0Z>|Nc(mK8UN6F7F z_p{|Y>vmX(5>LkMmNQ-TDEHGnCTdR3ysff5y$lv2{@$HRCrUU)s(4?E z_iVjt{>XP<#Z5B>l5%+t(@@3?N6`gSQ6E7FRnPpB-g278jI+j!!eL6DmYBoNJ4PEc zH20mGP0_BM5Wy9}U^CprJ!7UWcAVx9>sWoaM^Mhq;3G0w`%?G7GRE|%q8 zqJ?xMk(*cFOgKH#8`&HnPw{!^zLH-PyQ^ zaqwPLQ}7RGjcd6G&>l%;-93zc>s&B{a`Wf$wD;pEQQB7!vP&VMe3-=75x0?ksWBq{FF7M$nYXF?id zgB>a#lWxGRRY&vd-!d>dFLk-TZv>_##@ig>ALaMgKHAB`)YRFD;g8E7H8?U})^7C| zR|R%M0JjFzl-n&5%rvr4sN{wdZgDXeEjA{K-;tC`I9`m2KcTehnTBvp``=7oKwe|3wz%_0y40ykY9{avN4t4R?GNa%WwmVZO6!DXaYh{+AxZx zNgi3;D=&q2V^%x7a3?>0NwAi;P9f#T`wYJkth_N%)_u*s^{lD{{=xmT04Tc1?}>7c zHJ=SXWCoX2MlAMm=;|e2^ zH`*W~QmhTA2_uUOk#{GAa9S}x3Ri0J$?B~MmUauxYHLrtjvB>T_F3cCdUx&|yQA%s zRxLe-9`Qw5=9$tvk0;6>Ef_3u{PmR4VkP=tPA*?O+qwzt$aZCYNgE#dUH1q!Y(&ss znGPF%LbTTIiP%?PF%vgEUxAiRj2#6=Rj1lU~sTgs5YSW6AM4C5PYGcW z1GG3r7RS(+iY*Z) zJdbVbcSh?cdI~q%W9ef>5*2nPc#&EbRe(#SPjO;Z6Z? znQTg_J$63IQd!X>JXgd}>G=V^@oP$&SLxMis6cX|DR}BM@`gT|+uehzWbNMAIIju^ zyee6(vi9mWr(KWqp=B@uIWt*E6z6ud)oZ4V1s`MZx=8S*A@&N)@k4RnK-sOs7PD6# zU4bj^iqn3s(BU}azHqyW9F$*a;z5H%nM^^&MjiNU>}Vz#$at|-fr{%`;wb9MMIUbn z0(GJ4jZJ)GukhfirSR&ErQ*2Pob{hjYU3T!<#?+XlTgASrpK~?fs|-tzUGk^mnP%I&TE_%` zMpQkOgpB;PEUnbkIE~t{q%2LHD)Tgpx)SB6%+$v^=}Xw&{9>thlJ{g}_ic=NVpc{{ zR>+~|m4&g%Ir^a6`>V_QDSH^E80Lk=$xFZJ?*F7|_BqXD1|R`|c@zKu_3eo_?$gP_ z(9zUH+1b(3*8C5)rcz_oc2x|?hhf?evlOvOQL`S86&g-k5I7|HksJYqOcY=J$FR<) zZ>DZ3^+_eZviz0$vn|zsDelLl<)3ThDc-uHx<}#+4Qt zSm2bcv}?P;dlfkh+{DR8NWIl{w2|2`2GxLLBKYjJaPbC@L6c4Y<9N`Q?)5ueMo0Nh zC66M|l0-^jSaO^APN@ssj5{F%PV-GBy7Fm{XR$rbuI}_P?qPnbc%|BlBo&Azg>1{Z zYM+v%i0*d95n+|2Hr0bq-I-Z?3=9wK{PhG0dM}T+{I{b38)bOoLSCv%U`#7glp#i7 z4U-_3=;*al14*wH#Y!v@QMi{txx+55y{=nFxogX znSf<%Bqg(Vp!%xSoD|KhsHJYIGp+>4?GvBWW@?csWgyGZS7V`}SW!c~B4l#>b%j(z z#b!A`S*o{+c?2Zq`X?gG?G!VD?L zqCudPevU@X$yAPPJ}F}07dI%x+8TolkP62pTXoHar5|na$wG@ zpkIBd%rqq0!5N(T{#Y&Vc{vA(>qG8YF9dlcjLb9426I=u*{9W!7qFcl^u8xQEZHGG zDgX%FrtX7vGoi@^UwW7CF(sPUr59-(GXU)Pvfv-d(;O`m8qUODSQ=JjYZKClO?g61 zJ2MQIDjOTbQu$^@`U!LkvsUv3?Ch1pl5^$vExdVr8w~|p6JG7k^GZ^+a$v@z!`yiK zS_XD4By5zrcdo3jM5^1QVM{LZoRW@$<{Vwh-euJsja@8QvSOK^+qb6{YjRDVIl}o; zGbTm9gsgJ>XyX&li*mWNbxOu+H4Jx09c-8EE&MtV6B;EaqFb375mkUQ(Dme@@;k$`b9Nz2;P88D40Ew0U?#b{JSqqgIQPL=cLyJdfbyn5rFcxzL0k6Vc z4{X6Y%hhnV9_g$c4>8!*UAjVJFyQb^R;T-i2*aXEsN*{Uc)SG>1o#`A!bvukZ005- zC4Wb?)Ypu?Ztbs~P3+Hy-L-OlkbzrwUPoXM*DdmxgIY@xMm*iQPN*Pp-v<4enHtFjaeUyV^CR| zvswkk<@r9_&W+Gy0)$A3#bb#DyzAKLS}<2uAz)Iw%T!v)Ks!zc3z|_Ls4XhAI!V@z z)+)HjiXUw0Mac@0f!XfTzns|G#)sE zJz=$08S%c|$__+1sH9b2aK5B--mD{yOyPkru$>k-}A{+F;V}Ub9*n`$y~zj5TquXKY7{1l>MDk0dF&o3v8EJ${ATcJP0(9 z&X@p&6vy-a^l)Umzu;^Cy7&{f`}N`GmQl7Tg{Z^UGlQlY>TJK;H||)k2}syP!=jiT zYzZ!4w9OGU371?Z=)Csl7AX|^i;ljJ*SkVIac=a$^@Ba@hOle9fwclZCab3x(c^wF z>jJ$yWbqEVX^j4%@k95=Z>T*|6EAObz5lFw5$^_F2;YVi8F2sSPKEgwTNtT4 zX1mIW)QPdr58WxAXd>POKPDJQXhAO;*FDMFR1ckyFE#7%bgQcrPvBzsm|uj?(!kx2 z&fiIc)#YC |3{YhlnUN_uKH8zJo`;45lKKcV$Lc4#!ac_Lcmm$QTRct_QuvTa+ zZ0*>UHdnE-oez{|#%n@>cGNbgPVPXp`DREBECH!mT1m(9R11kAMwS-){c;rCY}O4* z3mtJI0p^hBRjo!>1LY@lt}o-BbL$ROT7??#sJ_w1#!Fl*kCo2^(~(U+XGdcZwDwP6 zb)kH-sGz|h!8^g`&GC>bK}|#dsk8~celskf8as7x>>{#ng#UC;W&n(tj58iPBlPE{ zWFklLcudi%c@AN?qZAaw;X99vrkWa%%ky&>1&U8s~#D14u~fLnuvYvU4V9eknf#>Fh^w5WwWhNhF!Waj|fs4cIrZ7 zpwNDmt4YPY&_KlWry-u>K)cF0E9jE}3<5(85PZ*Mp2?pM@dOn1K_)o``hD~XJbdrY z-+4>0cEwLx7Pg}M9EEuEourTpS4%q~G8KhnlAAl}lVBPBr12Z3dtT%{K2xGua(+Xi zFU%<_8TnrAT8q#tw+=%Y7a5JaI-vb5~# z1QCpXv1FoE^YKEIXg+rMHlVLFnX~zAl&~cb<{IzkY9w!$}mpFi5R6}iSv8GW?)h9pYCCNFV^{~m-iz(l{S{O{UW=ay% zYmfA%9Qlnrs49$W3;Px|Y!m=($V?RrlQzja8zz$lw;!4|Qi8zq$1?h2cvhv=@}w@o z_(@vq89X=>%^%zr`scq{?9(!2;LD+#5}AC2(cAwuhF30E4z(L}m*L`d5F;y8FBkJn2#hb44ZsDPf!IX_p)YoxBou;KK- zA3sdv#b;K&wb|@%IS2Vofns83tl(&8@5Eqi=Vb%uc$b5lSxEnP`m>EAY7b$8$Oam6 zGc4)pfVsg_wPVxAR9`bmLer-5%r9t1jJ1zjyK^YmFq7NtXV_OJjHHSqkT4VBCt=tl z9v^wXQjZh&i`6=JtD#EwY)1vCHwkB(FXPq3C8`y=Go-R55OYgerIKa@)%-=#_KADw^i{iwqT6xMye;oRs|R%HlIapT4KzRqSxH>=;DfY!CA>R^j^v1wCU)-+vRnd_SE8F+Pj8mz@W|V2 z8hGK-0*`sgNXh8D?AWuPHHfR?YCkmTmEwwsH^Q~sG=`6F)L%O*?)jSQ6bYhU^c}Lz zB2+?cvd8YT#xXF$Dg2F=R76{yx^YhWRp6zqscdJ0QSqYd0v>Z++77?`gV*_8iKy=9 z@KXKe;tp>vj`UX-H?+6^pNGG>_CJrzguhapIBm-caQ?^n(#%)22A|_y&E<-X^O{XU zE92fM&S=F&OGGf&Z45o$Et<4dV`U0FF~`s5hn~sh;9c-RKSN}Ak9%}|6k(X|Ko2*W zlqpDEGW?zlnwc6gmx;)Tzf&orIdx2@TL-~U%dF^enTEpG7GC5EYjQH$pe>6%0L>p>2Nq)?{-+TlQ?RX^jm;}NF)0)!%}LCVN6z}tWN?c7aBcHhbX0aI0)Z|hHTIDNHnZXf zJJO9o+94s=Iqk>irP$2vcCzUE8l;?3ArodT$lxu`%t<-pTh5#uDAHEkV+cqjOC91< zyhREAQ96+qjEgkGO+0(w>NB;B>qI+JU4osU*7ZXZ2)MITd%cM1PEJ2Cd<4uQCb6;U z#!Y!8?1HmN{u$v8MB*q0dL8tM>VVK7O!jtG^zklCBp~uVwe1y6+xSpoGTI#&Is?iX z&hxe>uCIcx4~sN^(WMsohXVaqsh2mR`X9|92q@hf{qgr=_<#M=e_j7Z8m}Pz?*RWU z*!`E`uWQy@;QU3t`@7-4ix~fE*!M=#{NIws-*JAIwEcG03bpI0Q^UP@VoiHXWPG;Q_%j!{GYj3 WK^o%CM}8^9VF81`l;RA(?*1Q1J*>t6 literal 0 HcmV?d00001 diff --git a/src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.csv b/src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.csv new file mode 100644 index 000000000..351a34211 --- /dev/null +++ b/src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.csv @@ -0,0 +1,3 @@ +Germplasm Name,Germplasm GID,Test (T) or Check (C ),Exp Title,Exp Description,Exp Unit,Exp Type,Env,Env Location,Env Year,Exp Unit ID,Exp Replicate #,Exp Block #,Row,Column,Treatment Factors,ObsUnitID,ObsVar1 +,1,,exp,,plant,trial,env1,here,2022,1,1,2,,,,,1 +,1,,exp,,plant,trial,env2,here,2022,1,1,2,,,,,2 \ No newline at end of file diff --git a/src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.xls b/src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.xls new file mode 100644 index 0000000000000000000000000000000000000000..ac5075c4a3f4aaf137a753f54e3641eb05e7093b GIT binary patch literal 36352 zcmeHw349b)^8cHhlR!wg!YMQmAb}i^aLJVu!jS_)01+@uk`5V2X2Q%QgoT(a1O-Jz zKsi-FkpOZFa^IpHDj<dyo6hM^>t0#8rRi0sIQlYI_FIJ)^N_(+xb7#F{TS|7R8Ud+j>9!THOK!$67W(( zksj{s>Ew?+k24qMp=RtVzBjua!Y5;2;T*&ASVP7VwHXGzQJ-bvlC!e389X=K$Q!sn z=!g+`(5Q%qgD|{4DmSKrO)dw#E^8w2R!{h>FMMtiYVH+2`>=Yf)b*&_+xj`tJRb_7*R8d34y0E>Zs!AoMVqR6$aAA$Apu(VeXT1$psaQ9sRNShp@po8+ za~xzJeVy__HM&$;qjwcGR4md-UplaCSZFs34j+Y`ax4MXI-X@Sli6lfuqNj^V$9ou zPiglp+?q3YXo$5AXx%2bO`A?>LA}NVN`)~&E!@(n_=&>j-^ViAwD)bfv}uX=}?XtO2Zw zu^%cp;amJT$PwJL2L2zF1Ibp!IQkL~7loYPGsE59w54l-Z>j`7yAt@)O5n>Xfv>9s z{%$4k%J_M$lKSf_fmepVvie&p!ShG+vrC|hR7M;x7QO7R1irlz`1_T>w^afc`)f~E zIqF!oQ;8|G*S{`W0+(dpn;X?#J@DnNshh(2YmB525 zfp>5SuMM06{_oA`nTX#0DevS>nWlLmiLce@%;rnEedl5&RDs5*OwovNxI zW?R39qx!(+5H6!@=2N7LPX%4EU(lukj|gwVSfMRl3tUlO#LqDXxy^Q)?dE4g7w~ar zxQ!eF1%B-71MUz%;11;};t}k}hOWH6T%K}%pnR%8RGn=Z@;`GYI2Xu3nm7t)pvb&%<~*f*dmkJUh&5IhD2V&t(X zxIJzI_s4BO&p#fECC|rgKuUQm3KT{js{zHA$7(=<=dl`4#CWU*6iObe0mYcdYCyrq zX@h9Rc%Z1{TKRZz69x*OMJ6ZCAULi_twchhsv1bh3@b-MajhCi$TTZQLP4+^NXT3( zM?#UY8c4{5D@Q`%vKmOptSd)Cv7kyMrxKiNhXQdmkT};4(v|8UajqQ%yXqiut{o&i z)j{H1JI$(x#JP4b@2{GZ@CfJH!E=G?AaSmp7S%)ITsz3zs>6wM?QqpY;#@nes)xk6 zcJL@g)*TdVBfaR^7;~Ue-$yuNFF?RU_s((MqFH6T)~+SwF8)! z3 zr+I-gPY!3Dg0)XeI-!~v_^1J9r?w3w8$0%`+{To-4IW#{ z+z6@EhAP<(0iC-MJgco5Nkyb{BoWm?q6{}my>`2ulbu%d?Y|eS=;$4V0#;DpC(ugoEB_;8yuuhUBgss`BA0g#&v|`dBz2bAvK$ z{U(yj0@; z+fq@grQe^;K>t6Rf%re0fdPLu0|WnT1_u4v3=FOg0~7|7K}Lkx1{qtJ=p98EnhS@0 znvno_HJp>Z%eEyZ#b9F85mO?0ai&A@o9ao2edtZZttwr%h(wxLR>gqClw+wVol{bE z82C@deq|(01_QrKEq`Prr6tlT)$&J5WoY%J^2bI}rK(bXFH?Wy2paa^R+VxDrB;W5 zKXM2UuMPwMNn=&c(no*+C+YhFmwL<((+d{fsVx`6MQ3F2I#cKQ>O?l59^IJ_m0ODn zA8Zrr#M&~Bh2V^48TjLwPS_qtk0eyAHuH4jvo2hHP~lG3lz3x))0P%PiA-H3nQ(hD zt(}k&pL_^p-1)3uuYc~COg$x;2zxT^oRATpjR<5s_^i8k?>Z(^UrDBmJ(&(p$k5Xh zv(P;GtcnMx9Fu9FB-7QNOsEqw^vuOfM$Knky8X~GnTAR--R#MPJ0T-JMG=JN#b z?H7*8_!F5f&MmDa$aJZOrJYK6@7Onv$uv@uabRg}oRE=Q+Np$}PHu2arm>QY150b~ zgpAzMP9$Ar zl5t>Z5l+a+E$vjou`BO6CexhAbaQTL9LRL5hNXo^%w1cFZk|w*abRfyPRPhDEj;4; z&83ce(?Utcfu#jHAtSf6@QCBbk2@yQQc1>vr3E`7Be%5hh>8nYj>&LJG7c=QlM^y> zOAC)!cKB<@WLhc7IIy%XPRPhDEj(iF>A8-`)W$w*ERD@B=>2<%Red$9^ha-?fosx)id4;$&RX((2claZgJ zF=_R>`Rc#gB6RVusKvzJ(qyoF{GvvK)Jx0sNWo~5MzNQW(vx_J04-w*1UwkFq~Rtl zqo`0>UQ*kXylhGfTzv8K0BnDX&E_*Eao8+bvZT~kzlN)O-P9w+O$)&ajQ_HO+J3eWDs)S3^v8mBDBO4EkYG7JS%G3 zwXmhtkcA-wXZ;HEax^-PNpG0KrSgTQ`RaIER4!dB{P63+Lj0DDbgvWiuLmPjAoEnP zQwV#YTwl^YDcfvbUg#TE1mpF37-}zDysD-ZZY&-`(<@As7BeMWASU6egP&jEIm`fF zld08>=P*}LFzV_D$qoQRWcyk$Ivxy)YoCP969XnLb%KbFOwkjZiH>lAOC3xyWXR$B zl*sEkKrZ^fN|0AXuG)DO9UpMVB z{!rAIjaU}Ac(blhBpay9%ZDlC+p5f^0s1o#lFfrHlG3fipB7IgrO^uOzd8K!vVv$spiUNKV6Stz%O?+MqjLgSIT1?|Ct^wBL@bfR>{;Rt*#+3L z;O-7Z3_+g_mWe-!j7+r`nQE%mlmlsXwr%C=3-zSJI!sU;$;;l_RfM9qfI@E~3s5|J zaBU)>Ac2e8cN2`t=J^82UGgJhZ>18MSY6f!j)#rZ7>HdpqLWFC4~7k{Zhk}mXi7+9VnTqUCKzC-Flh5AbHPsIc;1kUaAwTqk~Fz|eIdG;dDY}_ zalA>R&B3$xB>g+En{1xbp%$ml;!L<6Z)uy(leYK*O-{bX1oxAn(Q)ZKr`2WTwuJkc`o!CZR2N!EO_HYbNmN=~IVxf)YOHkz39LX%9$D_vvcx%4R_?aX|GK+7Uh zut}A;{Jb2!CNq>n@2#DR9~h@*s!`CX5kF|;pv^4SjvK3DoJ}20YveSXaf%Tff-fC! z5W11C(@x4qH!`6)ZI%{40ngGKZ1^^1qiY&2&!8V~(By`4GPJq*Mia*uYVm3AzNIgL zaWO&MWUVQi1Aj*J8fbYg!@wh-gd8=Tj-QOC7NaT9YB+6fp59;zl@TCS1R8@NOaspg z5k4Elkd*{;8eJx*hl&hbrl3J1s6oW`;&=H0!`0w!)*| zfvG_p1SY1Wfw5eW9&S^xJffz}%Hj8kd>WKZeSLdj@)Zv zuoWn%&S=cf6>SDxG!Yu$HE4NbC^BM1?J@&^7KZG0MyZr*9CLDDI+KpBb4wB*J(}DZAPdx)MO}Nl#pz{ zLJh_eZ|upz>`lgCd}(y1V9o^Tpp>mG;PD0Ip+Uc+e`bEjc`UT!$I1(|XhIb^H$!*> zj5$}M%YZE4gK~mlGJ2zt>^eP1p8?ewVXZKWjBJe|)7<~ZH!-B9{d5KQn3oIdk3hiV zFGPs*lv{J~-V_=p;hPsmeI~x?0XU6+Ev^lKW(nB^C)_Cj)PS-MUSF)qvB`j$Fk7M! z!wFY}i?a7T0fW|3HxzJbfKZiI+3BQ36*D1IrDN*mmjBv6r zKO}xK1ei!soEi{MI@Fsi(6yHX@g(w|C&-BUBL!${JIDM|9|(WGATvF#Gr?gx5{OKM zIOGISH$u5)mBFomOt-_c34=3>Z$acvG((m}z&8SS{HU?+I{P)Y+1$>~N4xC7&oRCLW73Pjoj#PAezLQBq8 zRaR(K5`>|bp%cbxC^s@211=991r7>h5NZ_@G@Eu(wnjq~*#Yb!6L~EuLdgDvye{F`Bx^x@H&ZZ%;vvBh;*#qOF$-6eCIr(oA{bsvz(aJ*y)sV| zK+7sNW9Sx{nb;Wz2aDC`=*Lg7XkD5hgmTH!Ehs$K$O(Bk(!xwWORIxC&3!Z{`{vx6 zrVfs$6_kq-ui3)XN;y5TutCXqD2Hq_by!bx-x~@hW-+T&7@pCDtj!hXCJ9y6W6mU1 z2sO{+Xtn|Ek`~$nZF^3XIr?CZMv^I*r)7#BO1wei5@yVD$g*_Yz+uTfx^?O-Q3xD` zBP1~~BrYyBJG*CYZcn4JV|I3KE{z-q_d@8Nc?tuqV|FG?07*`wW0Dj`F-ma8x$sx{ zpsm5Sg*;3VL+e6RG{S`e-d`Nxu$F#OvX18zLQ0EoA&G|9 zBk*+=zrrh!SgF|s>RU47EQ_G&CQW#t4TCdrdvE_G_{`K66~ykio8Q4KUp&a4UA}vhM*rZ(z+aS;j1y zjgE!*G1Sv2hc=E4EzHe%_mz`LM>_fSdpO|N88<=(E#DfYZhLF`hjTw%HSOR>O9K3l z|847<;lDggIGY+%#`)(2Ux;}4Mz3pzeJwq=A6dM7&7e&WVmb!gZWcdw^_Sg~FKiew z`>#zK_tvcTUHtv-{d)tuKVLF%_R^HqQ@_p|pRl#h+p6YY&#?OlbzMA*m?#OOQ?+%Jp&)9Zw{<%A4KleZG z9}{|FSNk7A-|jej<;SBAr~cez^UY`Yu4lTIUFIItnp5w{=GwEGhES8O}d>C_v^hl)1^-{|M2>%K5Sdp&(+S=hB<`{tGL>&~?}y?y7; zpFcuZ>^nMf%4_~J(7_jQ(r-sGMsoQu*=Hiw5-{T?3#ol7(fbJJ7j?C0<>Yx6x@YVO z?3cdvnlrJ0XGb(twr_T%g$Dy;{zdtQ_O|!Bu z1~mU5IOpnz4Zprs*!cK+FV+8Y;77)xMQzIG&D^m468HJ`j^(Ys4Zifuu7C-BUW^}k z^RtIRU-i4)v7NU|568?Woy|S-DMrlZmWdw9` znYZgUdByd1!}eQp+N7jhA29G-XvjT7bV`A#U=?UjK|SE!nPf3{@? z7rpGaS#@@u4GkLloooMB{CBX}!9@|j_K4^e(QE2A_qNUYOZ&yO4u7>fZSU~=eVY4T zm~}MzQt{uvomO!1VQJiV>Y@JEc1IpLd|=N%Z?&I%dPnU^rG4hT{6V*MzpPw%{KsR@ zH9j0(@BOEyXZ)0Vs<1R*%!6?mN6!S^eCg(~td&D9%<`LE6xP3PS)b-FANQMDwEFCq zqXPBAcFtKnq*m;Sse`)i2;!w>xa0D@1=HF)M3w(XMI)}8lK;{qvPE{ z?z93ew)w=8vzzK%+Av|_`s}Mdk4AS~KKbN7GOl(FEKhs-(Bz(p zn^(;|zoJ3fGlv#6+ur}==bNH~_HUiO$-iLq>UFErX2cBo>bUprAschLHd^yBm(l;> z^7~KkI=5E$$*bQ4&U!9*_c{Lz|C9M!{8wfi>&1pdy$~?F&y$1iH}+}u!m?TORqZeL z^@>sNb^9RUph2_kbWqNr;va%;>A!d@yIGsqr@wP)m(CCT?z^M&7Q}u2$a6(`cTWH1 z-Y2L2_OClX>&J&Zv%F6QzyJLa{PDXhdvv;Sx#FiEzWlQI>Z2cS`+l4D>;m1sIgQ%= zcInf!{l7eV*>~;p^}l(i@Rz&mCvQsI_vqgG$l|};I=%nonS-Sd2bRv9T#~#mK6tuo z&!4WT4lIw0j5|6%s@2VcGTq73E-g;^bdtgc6Qi-Endgwe%7b$(XA5~)qb*mPVl}pM<;S$bs6|(?RTS2%=~eH+p|k{si&_G zxoCQ`(5L^{HM*Oz~M0Sz7yA!PG^cv~G61?UJ3Qdu@&-WWJcVwABms8tsVx>h#ge zjq0n1UVozVt?84O6g^)y@xJQ|=Xd|&b?CJ{pC-G#+-ld-g%8-!0|~p1EPB{!+gI;* ze=DPO?5M0o@goW@-8kChiI=qFX8zdsScBJB)jf9k+AnoHd|rF1+ouB~)veaw(Tp6| zC#vAxb+;SW%3hF~wry$ahqE`w?0DXF$8*V(cPuQ-3Y_bgw&rKHSNFwUpW&Y@8RUB_ zFWtpg{a&$OSkha~y-#c09zMMxzSY(Vi+20pD|~&@$>K)4KD@HO_m1o_SN6L#nWk+r z;LbazJaa#bKlcUaySVRkk0Y<@qK_JiLs$J{o&U0)H;(<7xNMKAX`A(5&K@vL_s?zT zeq31b`i>7Lbsv*_H2y^L+Vzbtmw&Q1DS1=GI}KH*)5BVG4^4@a#!l^>zq`x1 z&@0*(OY7>odF3{_bY-?*i_&k-tQ`>+b8YMXjWq1oAt!XH+K8^T-knd$b;{nJ1{6~;Ldp= z0f*0B>3Qqz3ta1Z#+T=p-CKWcJP>@+YU$`n_3IsuAH2~+dn2QbXVUwx&-pN;_riBB zzSNMv^?kkYX-hjUKQQ6D^)p@#ST=pn)T=koH9H+tT*@6iQNAkt$}Dx^q#+3#o@{-; zUGU({DZhp^d}qoRUDNvAYCPzC-NcwKn*y`KldtxCb-d~O;aRTNCOn(fsZ29}N~`8> z>kcmMzkQu{MAwu$6W8DSIp&0+|JhR$-+6U-YPo5A=ZVkvYx30A8|5$TJv2d+{*g9o zX7aFyh7%h?43|>goHyL7Q(;zQeEGhp=uTxfmW?{s;97Z?cMU&IE8qCUi>`0P-fi9Q z@JIi8b>Y7VesvdUw=J1w_A_ruAiA6I(g9Q(;G`K zgvOut`@xhvf3;7@?tkJzhJ*O+b7tO~J8-8NqI-Kq2o#eRAt=hCxH*YQ^gi6FLFvQk zlhu0tXS@AgX?JPX=?&*xORsnJ=@uC<^v+$Mjlt^B=QaD6X!k67{doP1@XMXE*ELo5 zy!Gm&rH68N-kBSf;-578La)rtmf3)ZHfQR`-E52K%n)X4!zI4B-<*dnTLzY%-d(@-p?D1hO{m1?` zG}?G!rhDM3W2dgPS+*|0?ZoKayN`Zp>AK1#1iv+`EUQH|+2J6^9WvM!filgrw(bOZ z%91{1LuJ}zZP*0)q+K2&6TcFZGKaBZ`rHvFA)`^^P#S?1#|6wIYYvVH$E0A0FQX`;bZL!MCSd{9o zxN`^-9!Bp+W;2A^rNU=B)Xox)uOsPorqXqw13ue`st?1p_I<=%ZQ_aSo?t3`eR2R5b9LA@8Xp?0AKTe`kZ2=VUbNJcQypf|gT z38xiC6};Kog8oa)^{9e3D-+r>m9Nm2WbMsT2?c12FwB7MV1!877E@z87zxmu3GWJm z#6^6tSUY@VT}pg3E%Ro-3jL6Hp=q@eh6PtU7|F_;{bZKQ2%&EzM{mZNo8CxnO$^u_=~Z z*&x^iWBc)5Z2=RDvix}ob*N=O&=6aYh4sUDmP#%x4$f05xuQhfAZ%pmR$NiAG@?k9 zqlK{{mQ*N7*vML=X{VC8rYlO+8^Y$|5F2qGBTT0W8*RPmZ^7n@Hdfxq@?(S%Lp7;4 z(X@>~phTMutX#O+aN&k_(b(%C)+A2j;a#MXoF0Bk0t%8W+y#21ZK6leX!s+kB&X+L zL(fB?N4lVPB+F>w@rqcIxAU}V=P9%!YoMAm3ZjLPE0*N#)HdzZLOYu3QcaS5G`$Jh zT#~o*vT5ffv}fDO%+kEWz(*fO}kn` zJ9?f#HEC2v)61sKB{`oI0j%`M=SAbq?_y2Tcr?AlRA}d8LyzKzwVjX9jx*OJ{rAU1 zFR@)6w6oe7y=lI`MelXcu0OrtTCmzWu(aW_KXD9@{q)B^Y7v{S4K|8gR&2fkwg59W zUx(Pp#$s5Q85?^IOG$;2G!iLt##%;VU9^i8azT5JcXQ7+5iiRjP)Gh;6z+jx5Sds|CckBM2r+y zjyyh{nK-3Gf>AD`!W}V}fnab*IgIqJLLT$`a+z}P!2(7pmnrv7u#_n}$nOiejO@M} zG$_CC4(};%uZCAFS0|7%gOJwC*@G|oDJ`^uJQ!VZ5AS$d=Tg$$% zoDSe$!taO7v)nty!av5FO5#t-t;i~vFruW~s-8LjC{ZO^a;thQio`F(cdADeM)sro8iuU#;TcQtb-|XL0q%h6n|W}t;eqy*T6w_#VL?ga!39!Hz&2x% zTl%jcRvuicmAEO3wAM}3WN5K~Zp`NT z6=v``xEFJ4}Zq^(%5~z_tjRa~WP$PjF3DiiSMglbw zsF6U81ZpHuBZ2=N2~;-!r}?)_`N!qUL!0?6ScLh1@Xs6L0CQKsp2A$6=IxH23DAlN zJrkfOhxAN<)^HZ$q_v|JIBVf7!%6e_%{Xc9PtOQwrRg9}dM5A@PQv{y&gwi9kk9v} zYjIBQC(r|$O21ox5u!mWYU1ye1=g_pTK+?2R@+>S9vM&M?1l3{!VjzT5qTQKD3jQ=Ki#%PWv%w;!OK1Ih?I< zw#L~8X8_K&INRaG(!a2Okai8yUcrtygK>u7q+NeuI6L9&jFa}sMBw}ld+lgX5AD{W z-7-CJK8cg&-FOv|a38;ZW_@u+;*7#cv+Nk0u{f#yBb@a8|I3krx6|nHCLIntw&`6! z^ghnVIsBQsu*Z?)MRvrg=~H}ugRlNVLGeNIl|Unfx8qF2J8g)Mi4wTl-`GuRp-((i zx8b-fv8FqAlhIr4=!L7)_koat_zon3NN*Ub9BUxxt8kmfPj%_jo-|?b;L9C*Z|R*x U#Dn;brN`r^$M5KWZyEjn1G?QvdH?_b literal 0 HcmV?d00001 diff --git a/src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.xlsx b/src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..882be4a57d084ef0981cfaee6b297d061ffb770e GIT binary patch literal 12786 zcmeHt1ydbs*7d<5IKkcBA-G#`4i1o)!K^qGbA{54gmc1{{J2S#c!Z5Vce#N8KM6){u)WOLK$zJS?+C6?=wDU zel^S>t|-|Q`6tAH#%w&jv_hmnp``*vyoU#Gdqd=b!+sOGP_s5V@y`7)1g)KV`wOKu zh3pI7SQ46|#49A}YB2Pz$*Wnr7C2uVQ(k&;IrNi)zLH=G{S64N)zrHM>dkqsWMNhK zcQD-hS2o5w1b%rL8i*{o0zI+}48{tU&utW8dWuHK%UyG&=(LcQwBoX~YVt3jAr~Yl z(?aYrxh}kZl$z<>mAhrVl;W~yi$~stIzJ}k+3j_f8Z^2ow%1n0o>T$BKu?PPeoVp-mD+eaVU+4b?<$rM` z{^ilj;^Y*1nGu3dB_BfuZ)R6wP();0ge6-^RJ?tqz9ZL1=8@yAbW-7?sNe=by!UDM zejHw2;fp*NB)Q&XFAGOQ=Ot@!DGyG*w|9b}r358O*q3eep*qiA&0eRzmv*CeZjYud zZTgZcGqgq~F?}XhjWW)th70$$06!R$Kh;m8Urv41;I;y6Rz&%@Jh-ZnGv^?FJl$tL zsqhD)U?{ij(NrqNpo4+wVuk0R70J~duBx&bw|SL87Kn$`P1ne(>q0EO^X-!-gM7xI z5-B^{E$jIE0g9X}AB{Tpvu~MheVlN;rGr0*{KJX9tiIN%{|pl8;zIrV*EjMW0sz1T zz=FA2G5sS>Tx>y>2DY}AziQaOVg~HBkiDY(cOPwWqZWP42!W?T9RZW=_Hoa^VIB{f zp!SAk80KoU5IHNuVnMfuw1g9ms{G`MJaYC6mJyQ&?#oRAGN)A<>Kc5=mZkKu`Zqc^ z3TijRq4jJnH%J%}l2zr5OdR72mhuWltwN;5TuOSWy#Wa-n)vL-0y#3t{R6@dxQ28A z704~(>brND5uNXH(#$_)Dx$dT;`VtX4BgpUud!=KMt6AQO`PGt4~4Q#c%WuK&Xoe*yF2_7s)h3DnGJ~7_c^4_QcGhj=~S92=ElJX);kb)Y2kE zdmY?Y)3J{VUV}co9Kipb=R9dAFHyaEO#%!6fc)wbub%T)pUGF%wB6)F_trIk5#7F- zvG-*UON)X9LsV)l!7f~J!%gi&QHbZViic`(4;rY_vg%lqUb=DDdKuBkl}2EGHb8aPf8*8kbP=7N88c2tlEnb{q7n!y zKSVK1+%3yLke(^?kFSrRn5|00E3+6fD+KjBs_V_$^JyvRIE=`-Az9cWM4$=1sORLi{XOMQjY6RSXi#1Wz&s@o2+!n7iE-Iz+&j|^X77u$ZZ2M%r2FyoS#1^dn+F8h zm~EbLzm4r&n_jd>p?Y1#%jW_qE59;-TK>@{6ZOCfeLCl(Od$@Z??1ui!`K|b18t*^ zJYqid&(+afqPfoZ27KOwIvF$*As7fUsrj5HCsyFc5wiKjD9&>v#poL|=l?WqHPit5V3l#+p%m+*TGelP#M}?b*ZT zF2DcM&7PK~r6<|Hy76>B(C53xdbD_%Qvh}727=kS6Ke$0uX{P0wR5ZVzTD%+`=#IJ z#q!%zb`ZFV&LerU)`pzL$hH$-Lswi7e2&pcd9$?tC zOoX$xz0Q1A5gAj6TT-7-7vt#0&obX`tJjcTUP5+fz$1TB^fM-{By9xMXeg#vWULlx zL&alhP8M|0p$<2DxxKW$owkf~QeIFenTs_tvjNj*@0HiB;Z@|w34!PqsIwJI#2e66 zAHt?gjmG7S?;+}uFan)t87Etqmr9fBHrG2xT;Wsw;`M#SM1^w#w~(&I%dCG<*d4v% z#$U95RgqVVoHC*JIGbjSkV&O2ov~A3lu?y%ZD8mY^mJyx@?4YgyprDag2Nd+C4_ek z)$>GjaNPMCG5?v3d&GBnr62(S155w_@gFhe;OJ&$?C>iKfB&c%oyURhT{-jO`#5F? z$BYtG#t|P@H`P$Wu3TJ+l+veRNM>geUw*&l5kaC)880isdR)f@$8qh_;c&)%8zE5c zV8Rz$^Vxr3u(>)`*p*ai7>nip$=)P*@C*4ajfsCewzXuCK%~#JxyleC8M2H>J{U^d zr%9z@12c8i8glLm8!=(7wjr@pL%3d$zN)yCokp=7cdgbkQW?H-6nhV|f;`xe{hhUH zO4uMg{f`V*TF!Tvtmj6z#@*s!dk9}D2#~4zm*Auu6M+iHa6{%wpQMe>p@kS?joC>a zw8-dc>}o{87rZ5-Y!@p&c=8QHCMU**VkEPi1GHlay|*@bvy0W%-{z}DzMnPk+*q6&*uT{D$yhm*dmt3Ai5Sh&I11xBAw-(ev^P@XDk` zMbYLL^kEq6TQwu#Td?JE@l+8d3*>&l<(%>sDp_hE7xfQ=;^zYDbr;6`z*=6y0(2(G z0okvPt$D3)<8Zdiv4<$OGqFNoLD?Jy=4)J8tc5Ozi;Y>Q&^BIO^c1bAUO2$f4oV*! zB(pVIADkGDb>r*Otz#9uA76_Wg)8|L)!J`U1myIDQYZqeVK7j9jFvwt-#KUEcqWd6 z?x1Q20)vL5j>$3-b`=K_3BR}=el|ZsD#N?%c{jKCp{uzJpPFja1_BFeS@uAARwZ22 z3<7xLM4+2fd_VKnGFm;{30h$%DVS>JCX?y9T0e2~4ox#!kga8%d$J_rJ&@}P=!Gto z_Qf6EFC^lWf{}2jWp+WsbT4EGHRO@ZsH^@(j+fv~H1c{+DNuQFix#Y)L;6{_`Z>uQ zsh~7oc?6H84xQo&ELENYT=iLziiJFeBlOu@V7EOB-Q)fovd5I77Fln-lw8*dj;|A! zu!PmaETj$U%)@3ypyx7D&nua9maX?{`<$`zo1j5z5_b>CgRD`t6jTyi%4Xg z@D2hrcOg5!TX-D@J|8!>vbQ*D>Jh@C!bv01S#nKq2pCgK=w`S})I`=RJY3-){$LGV ztYLqJXh{dPP(a>GM@iJkWLW3%9F*^v7;v1f+Br!@Vi}5koYh%!*!h+k=cnRxy}u>* zBVK7-|Ni*HkTrXIcC(3@g2*%;%38$Z(9C6f>yRT)xYc~eL+SL}wj8pLH%ug7c~`3p zCOGNwjPcHqTbr?+pwqlpLcx}_Ph#;Fvb_d4Od*z8+5sUZ*a)5uySyjtKPhjXM^5c- zJg$kYzu7MlZZDmM#O($uuHKhi0bMXNAbZKAKbBLKX*;++a{g`3>Bd6okbkY={lvdG zj6dqQqnWX_G1DJsmS36YNPRRMj}yHU>s}DS!R7JYRy4)x`nXm63b|2c0)BnNk>Uq7 z*0?sjx8M}qS4!0R1<4||{0UP+5Qy!U(QxE-KZqu&MihMXU zvtg_@#_MF?5AiwW~qjkZi*vUCt~gF;f=5$Mer07a^VeK9~D@jv!MwefXYF18SzHIqlBR!USueK(A7#&+RZFG>6M^QT>w>`MGab69KGM6Bfr zqjVYIW{)jLZBuD6vdVZD@e^8DAy@a5a%Z(@1VL%5Xz0#7KkKG!RoLYTN@YOkG=(Tg zk3xyDPShauTaPhu;L`c%s&!@W7t2|GPa`t@*L5&^hZgVje=}oLJW}EoV|ho=5pJ~2 z3i9cGPg%i4F6RD;=HlLvK2T+4(ZoI_Tuuwl5#>UHYQJlEcimmJz(U0xXDB$pMLZ9v1Uh||Hsa_(%k#rf|;F#)F zh8~!4T#4lz4Ja9w&iLjm!Me)S*52P0lR6A_E&mQzY(-j|YbQg+Ju?bPZv7qj4$*e_ zG|fEJK66}ar_^C2th#BkJLZX}wjG_fZS8=b?kfLr2PBTC1rG6bg3SvCL8ptCNKEg>(*N2y}@L?Qi3^BfpM^)lfaU3=_J5IY2{2Yo0wM@T5~4 z?}_=wn%OB^c$u; zJ>Sn#Xsg~~BS|><2KBX7#KjA;Z88Fe`OC6m3stiP*cj<;+Pf6`?+IiKTlSQKRIE$U z69^ij#w%SJpJcC8T-hjd4D&;r=rFP;${*w}T%#kWr?K^AP4E<*uoZQYM9_6i19cWtKTbM+oCF>u=W0mU@4R8Q z#=vmhxn37<-w6?35)L-OPuMfWD5^_si>P=rmMY;M&sLlY1~gz|BaU`dUTm>T^zqu< z)my+e(vq=T-o4$&?wxmqZYG=2#QLn<Lh_DqF)4FpH8TnF5Ag#{UT6-9{`u z7oDl0EwFi#X8=`LGBzWj&ALgqZOnLuLv%AUWRI}Qskh1ZeyXiiT&`oIhec4B;kdPS zlHlS$tDq7b%pB2h5~4qp&b+-J{@ylg0^{Px=Wgf2Riv~J6tPVvqrRWQ8%*HP3;!Ok z9tpeF#!bcW;!R@(vm(8~kbtY-iJ}sql#AL>8sv=q;oDP{)`1f(^15EEq88Hh;XRm^ z{)D)*dF8-71JfUuKWcD9oSg0Q zFRBXSnh0UVQeA$xKsdv|3|PjCAlm3;Dqd(v61O8I9e*?*9d}G`7EFqzOtjmZ%v9gR zYnIsN>oiqIp>HmMW)@Uz)JiinPK3$)8#NL^D^eR?^qTDO~ zKL1U^t5ykTn|yP7M0tnS6(`P~eI!)N;(S2~YN@&uK^qOjI}QCc7>*qSqk&N<9U1*d zwt5BB_a3>){A*L1pYnG;$1Mm~@mB+rx8J7w4B_N{6KCI7?^(?(j~DFUKlO)Uh}cMw zf2i!%f0q$lTo$<)d^u1;j)Mux5rXtVi`sxX5#BhZOWHE>8CcSf4Ej-ge&?=ohC+2W zN@;nrzWPPC<{U9eoB?CTv*>Y=gqJ`d^~p3Qy??6`3K!cmLu~+q%MU^<$h?dpqDSOb zt_}ei^=U5z`^GENGtL`U;S8Ig>$AaL+hpFSoR@^i%&%4vq8#Z3Who}&-tP=qnVFxJ zfo%rF$A~!7V1D*Oo+DF>95W<~|K@(;Nx_@+Hf1ENt9)p&c zQ9LpFkx^o-!Ht;NoJc&|phQy(1<`m?3Xhj>jBqp?>6V*1AhCQ7!g`J-`7bCx0>e8g>C;hqVs5xp6t>g&||YklmZ4hvQR{@XF2it zN2jEZqH`GJXT)K_R`}#${j+10*b4@FA%6VONIH7-{&OY6wE`svDq=1m!IBcWEOCcu z%Qv4{4c?^j1ftg^UE+`%_aUG=!73;sSg};~b`e6rFy){pP6?FkdoEpQENQ7HxNq+f z6O#djB?q=*jzrJWi{e0p6nbvaD(r|Z1IPyR7Om5gp>x9mecFeJr!kcLudw+T3xx%1G76h`2~#eY?>u*m9#M%5 z2Vj^b$pONYxtSVitM@q@Od8E^r2Vytx==7oRXxQ+--vD|^>dopS*aGbJ5Latl+Z5R z8=J@_f3d^;jJ8l*@PNn@KJew|4?@G26-N%_W*DHEt`x)?6E_o`A=d*4R* zRk#pUDXNq;m$$fWJ7o^cgNZ0vDMBK-x1uawGGxsJn1ffvg4gwNm*9@>3w!#CZ|pZ& zJ#rcHobi_&_H#rIMws_STU6v>e7+?7sFf_1%`082hMbBS&Y%DrDU{ArasHMtjJ|Z% z!yke~o3DCp9aq;S+Q00sVYf;%Djo6zshM8qAr_d5?~!{-yu;~ zQEuhGrU!;_MO2iV`gWnX+swHmsASu3y$4lt%sn^2{~1l61{Mv*i(KnT?x6&wD8cbaPmIGWNTOB zi0IFVs-cmTRk)I)mmVLX`!FaaM^~-NI>Dxpq&AvQGKzLRy#SlZ6{`h zmTuwT#o;d4$>+*SN|Yd_YJ->^Id}A4!Bp_s?!b$<&GK=P8a|gVRo*MkKKHRN$;!^u zTss%Mqvf!eiJ)}GbR>7MXdomcx+rMML@caa)^|;`sW{@fbo1hVxH9^Et#YMlk-oFd zMG5mIIOam8eD9GXE$2ZxQNpxEEZgWj#f@#pPABa#*RgJ=UN)CDbl>;Rs|;<$-C?Xt z4Gf5YaT^)u7QMGB@>uu@V-L_e%c~eeQ(;W1{zoK;Su3C3*Shr^t@|Ctfj@Vw-fA<0 z6xx*B3c!mJs72u^t>fCH&$QETMfA8$*I5`!CfuGR_PE#wRT7(3e*|`9WbQFB-E#`o5XI{}Kiml33vYtgn7h=uas)Zx-6)cVn|3nNt;$?(o)W_XsUEkuUU9$b-Chr3 zjIdG&na4y>vw8-oEn7@W(@lw+Yo|EkOM=}z3P`V~6c|$nupQ1BiVVbv>k|~9QWCBL zQ}h)Z)HHEaAp<0ZPpDeLN=M+O?)g)pnXv%kYrnsKKc z`XUkap;vzPX`u1+l@05x{Mk!E8%b>ganM47Dg}h0_b}d(KAcQ*mC#LRv6sAl*cj`Z z_Q>m1TPQXTiL!Tuq`f=(ko&Zljl%OT=d=roG6GKa32u$GJdFUBpXPeRcc-x#4 z$`=SAav8r1*3N*X6n^enyu+4kSe2Ql2c-iz3FRPtDN{k_@wKP#pV?}cY#CnOe^D~z}3)$K!hb84G?3;P=dDrO+H^sl${>&{((a44yi3)S! z>uK!UHIuYf>e#uox)iHumVqxi%XLUR44MWx6~D=>Jp6VxYtD{i`qQo@rBIz`>=cCH zP0O4Z^&GOywcRZ6J~z_o+{PgZr%C^lD|&y6R9F67Uvy}syoloQeT&^tTH>M5C8gm7 z#;3Q36qn-U%Y%~yts+yrk0C#KmB1u(1aaq|vMeiYacwNGN$z2?1WCs>`c#mbK|rX2 zazhIVCm0{(na&<|cKju#5|LC0MN>ub;ONTf`gJ>_e$d}{(vrdS z5Y}XTD}I0mS)C7guwx0Pp7A14cKXOHba#&j$0-_VBuEm6q3j#oLMt-MhX-+3(~07!u+&Uj7&dTNZ1* zfs!BSL&{6~UWevfTRrE~K}VIm4|Krht;Zn*)K#MbcE84gqyb+?jsrSa>_(qr5;C+y zyWJ(Cj~n!9K~$#;k)HOd828`{#l3`x zk4*mQM9^_c_zX*~V9<{(5%L~!FqV3qRgAt!SImCjIPe)IV4Di-r!(qW8SI)EJiW3n z*~{fHJnnC^Y+Zsfc|5e?Zz@ndYZ9XIIzqLmYU)s%VQKB z7)`?S?1aJQK4k9;^91@fjZx*F@G6h7weVJkY8D{Z$^yBjI9hL1X7R|8e!kx)_!4J1 zi92esQWpNU#ljX$IjE>fS9qqVZN{WEl|x3WR(Xij?Iw<9TKDtNroo!-2hM>Q3d^Rj zv9(N3k}&PYdZ$2@pe?Xt=b@|R4<1q>ipP*k5 z-H9v7V;NJM&xgh0@mc(+ z7s4t}=N8p;0zSv@d3ge>cVOhe_2D~ur}XIa>saqUt6tbex-9+caAFqW|J26b7(=YC+M3~jc=W2w418bgi|8ye*1M-Q zp5?3rNtA=ACMO)nObNy=o7kp~vAy?j*c%AhKUJ77@(viokA2Sm@@U?m>TYcTz#|ql ze50hfysK#R5iu?eSU<5Ym^c-u;W<+}Schx&Ls&`79N#oHzm0@IwBmFFsQMMBjk+Or zVj)T_AP4g)J+0A1FaC&-N9xLbT7a2CGyfx~`*<}zPN?S5dX`!)8GO85rauAk`}2ms z4yZ6uF%dDA@gPDc%MDkK%qVSKI*Q7!8V0w--&5}VYpB>o>i8a3NICZ+OcJAyFTf8KScX{M_d zt<%liGNG@VI`=c4K9NX>o=M@!<|@q&OB1WIjg>cbxo}E~9VKPH4ME z6R)SY+qg3lk&+@QIVtN}H=oa+{t)W?(NiTPp2`O>qkq*6`UwCpXc81gqhejuJ6QE4 z9_p~d{-nIN3UA4Ca<$Z_VEA2EUcp0Nv1 zZ~5U2QHgD|`Z7VOcOcTww_uMLgC#m0UPX~5>2KREZ3EN$u5z3NL&5kR2(YPLz2uFK zGUFFY-_=aPmKnuxOAxZxq!0`HrY6M3itWWthgHe_Bo8PkjyRG zA)K}CU~e|Bzy6F;Vq90Xu}LV^1N$!622jSCWHIAhy=Qrz9U0<&=o|`(%6XCMk4*$9 zrJDqp&RA(Ws6il%M)r?Q)ue71DS}{edL=>YmhuBs;+xo@5Kt6J7N?nL57xXWZsoyT z!$uDWeKl)C=e(%Y7%J4;XQ~viXEtoMI>=t;PX7%@GiH2IjVuQQL}=?!xc-0M4JU!j z)IEW>rQHcH_-8xJ7sZ0qBZL6iX>bPy>4zf6h{%pEDcQD8EI&q?p%4YueaI3zacsrX z>Le96bn6M+Ctt2WY!a$6&Gl91h&#1I8e7N*{h#;%vh@$k-LVWo&jeNzgT!qU=i)cx|P!t=!j${K8L1KCv5F@KUu zc<2^<=^ZahJ=7i{(t@NQJi^XocUw5)AHw=82K*8uWF@*$Y+x=i#%DWz`m zMM7-?D0I8BYGKi=@YHoI#$3zCbO_XD1CF$Nj6 z@gNX5)EDpfU`q7IJ}@IxMKhr72aXSp-z8VJdVJVbXH<$UAYBU+uhzm`+_ZOH9eqSu zUNDoFc&X%4s5PeTV;m2^`;3xZfj$my{9Q$=sj-6SxL4&X1qXVAnM_<%(Z}wN$rhcf z^MC9}e-|aH`Pe^Kyt=sktBa%j)y4Jg?EdHBude;iBP0H=bSGZ(A`rpv=(7y#WtCoc zoU^HXpV!@o(Fx@@RSMIZs{Ha&1pDuHZA47v!BlzmxL{MBj z;>SI1U}9M`re$7{p#+zUS$5u*auzPX+~}A{qzpGHQ!XX@eZAfZ>iv~ar-hh|trm)?yGoSoFCs>)8qmR;+!Xj7zV^<)*rNY0>iOOD_i6l}rhO=XnEpPS{~h4>dBUFnqOaqHe=ht#rVGEL z{LXOwiE=~mk01qyN*Bh4!@)Pdq?mm1^}Qy0|5L-fAG8czh~RO gnzPdX#r&VS7bpYu>Lb6z;_!g}SJ5~o)31O35Au7#r2qf` literal 0 HcmV?d00001 diff --git a/src/test/resources/files/experiment_controller/germplasm_import.csv b/src/test/resources/files/experiment_controller/germplasm_import.csv new file mode 100644 index 000000000..a5b424b86 --- /dev/null +++ b/src/test/resources/files/experiment_controller/germplasm_import.csv @@ -0,0 +1,2 @@ +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,ANE,Wild,,,1,,,, \ No newline at end of file diff --git a/src/test/resources/sql/ExperimentControllerIntegrationTest.sql b/src/test/resources/sql/ExperimentControllerIntegrationTest.sql new file mode 100644 index 000000000..749796fb8 --- /dev/null +++ b/src/test/resources/sql/ExperimentControllerIntegrationTest.sql @@ -0,0 +1,37 @@ +-- name: CopyrightNotice +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- name: InsertProgramGermplasm +-- insert into program (species_id, name, created_by, updated_by) +-- select species.id, 'Test Program', bi_user.id, bi_user.id from species +-- join bi_user on bi_user.name = 'system' limit 1 + +-- name: InsertProgramObservationLevel +insert into program_observation_level (program_id, name, created_by, updated_by) +select program.id, 'plant', bi_user.id, bi_user.id from program +join bi_user on bi_user.name = 'system' and program.name = 'Test Program' limit 1 + +-- name: InsertProgramOntology +insert into program_ontology (program_id, created_by, updated_by) +select program.id, bi_user.id, bi_user.id from program +join bi_user on bi_user.name = 'system' and program.name = 'Test Program' limit 1 + +-- name: InsertProgramObservationVariable +-- insert into program_ontology (program_id, created_by, updated_by) +-- select program.id, bi_user.id, bi_user.id from program +-- join bi_user on bi_user.name = 'system' and program.name = 'Test Program' limit 1 \ No newline at end of file From d035bbf11aeec57d1427a4d3a03e33d154bbcef9 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 17 May 2023 11:28:31 -0400 Subject: [PATCH 02/27] add download endpoint to experiments controller --- .../brapi/v2/ExperimentController.java | 25 + .../brapi/v2/services/BrAPITrialService.java | 57 ++ .../experiment/ExperimentFileColumns.java | 63 ++ .../utilities/response/ResponseUtils.java | 12 + .../ExperimentControllerIntegrationTest.java | 690 ++++++++---------- .../sql/ImportControllerIntegrationTest.sql | 6 + 6 files changed, 472 insertions(+), 381 deletions(-) create mode 100644 src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index 787f9edc4..5615bff8a 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -1,9 +1,11 @@ package org.breedinginsight.brapi.v2; +import io.micronaut.http.HttpHeaders; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; import io.micronaut.http.annotation.*; +import io.micronaut.http.server.types.files.StreamedFile; import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; @@ -18,6 +20,7 @@ import org.breedinginsight.brapi.v1.controller.BrapiVersion; import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery; import org.breedinginsight.brapi.v2.services.BrAPITrialService; +import org.breedinginsight.model.DownloadFile; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.response.ResponseUtils; import org.breedinginsight.utilities.response.mappers.ExperimentQueryMapper; @@ -83,4 +86,26 @@ public HttpResponse> getExperimentById( return HttpResponse.status(HttpStatus.NOT_FOUND, e.getMessage()); } } + + @Get("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/export{?queryParams*}") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse datasetExport( + @PathVariable("programId") UUID programId, @PathVariable("experimentId") UUID experimentId, + @QueryValue Object queryParams) { + String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; + try { + DownloadFile datasetFile = experimentService.exportObservations(programId, experimentId, queryParams); + HttpResponse datasetExport = HttpResponse.ok(datasetFile.getStreamedFile()).header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + datasetFile.getFileName() + ".xlsx"); + return datasetExport; + } catch (Exception e) { + log.info(e.getMessage(), e); + e.printStackTrace(); + HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, downloadErrorMessage).contentType(MediaType.TEXT_PLAIN).body(downloadErrorMessage); + return response; + } + + + } + + } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 7bee585a3..ed9de26a3 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,6 +1,7 @@ package org.breedinginsight.brapi.v2.services; import io.micronaut.http.server.exceptions.InternalServerException; +import io.micronaut.http.server.types.files.StreamedFile; import org.brapi.client.v2.model.exceptions.ApiException; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.core.BrAPITrial; @@ -9,9 +10,21 @@ import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.Utilities; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.model.exports.FileType; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.model.Column; +import org.breedinginsight.model.DownloadFile; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; +import org.breedinginsight.services.writers.CSVWriter; +import org.breedinginsight.services.writers.ExcelWriter; import javax.inject.Inject; import javax.inject.Singleton; +import java.io.IOException; import java.util.*; @Slf4j @@ -51,5 +64,49 @@ public BrAPITrial getTrialDataByUUID(UUID programId, UUID trialId, boolean stats private long countGermplasm(UUID programId, String trialDbId) throws ApiException, DoesNotExistException{ List obUnits = ouDAO.getObservationUnitsForTrialDbId(programId, trialDbId); return obUnits.stream().map(BrAPIObservationUnit::getGermplasmDbId).distinct().count(); + private String makeFileName() { + // _Observation Dataset [-]__ + return "Exp_Observation Dataset [KEY-EXP]_ENV"; + } + public DownloadFile exportObservations(UUID programId, UUID experimentId, Object queryParams ) throws IOException { + List columns = ExperimentFileColumns.getOrderedColumns(); + FileType fileExtension = FileType.XLSX; + String fileName = makeFileName(); + StreamedFile downloadFile; + List> processedData = new ArrayList<>(); + + if (fileExtension.equals(FileType.CSV)){ + downloadFile = CSVWriter.writeToDownload(columns, processedData, fileExtension); + } else { + downloadFile = ExcelWriter.writeToDownload("Dataset Export", columns, processedData, fileExtension); + } + + return new DownloadFile(fileName, downloadFile); + } + + private Map makeExpObsMapByHeader(ExperimentObservation obs) { + Map row = new HashMap<>(); + if (obs != null) { + row.put(ExperimentObservation.Columns.GERMPLASM_NAME, obs.getGermplasmName()); + row.put(ExperimentObservation.Columns.GERMPLASM_GID, obs.getGid()); + row.put(ExperimentObservation.Columns.TEST_CHECK, obs.getTestOrCheck()); + row.put(ExperimentObservation.Columns.EXP_TITLE, obs.getExpTitle()); + row.put(ExperimentObservation.Columns.EXP_UNIT, obs.getExpUnit()); + row.put(ExperimentObservation.Columns.EXP_TYPE, obs.getExpType()); + row.put(ExperimentObservation.Columns.ENV, obs.getEnv()); + row.put(ExperimentObservation.Columns.ENV_LOCATION, obs.getEnvLocation()); + row.put(ExperimentObservation.Columns.ENV_YEAR, obs.getEnvYear()); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, obs.getExpUnitId()); + row.put(ExperimentObservation.Columns.REP_NUM, obs.getExpReplicateNo()); + row.put(ExperimentObservation.Columns.BLOCK_NUM, obs.getExpBlockNo()); + row.put(ExperimentObservation.Columns.ROW, obs.getRow()); + row.put(ExperimentObservation.Columns.COLUMN, obs.getColumn()); + } + + return row; + } + public List getDataset() { + List dataset = new ArrayList<>(); + return dataset; } } diff --git a/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java new file mode 100644 index 000000000..beb533034 --- /dev/null +++ b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java @@ -0,0 +1,63 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.services.parsers.experiment; + +import com.sun.mail.imap.protocol.ENVELOPE; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.model.Column; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public enum ExperimentFileColumns { + + GERMPLASM_NAME(ExperimentObservation.Columns.GERMPLASM_NAME, Column.ColumnDataType.STRING), + GERMPLASM_GID(ExperimentObservation.Columns.GERMPLASM_GID, Column.ColumnDataType.INTEGER), + TEST_CHECK(ExperimentObservation.Columns.TEST_CHECK, Column.ColumnDataType.STRING), + EXP_TITLE(ExperimentObservation.Columns.EXP_TITLE, Column.ColumnDataType.STRING), + EXP_DESCRIPTION(ExperimentObservation.Columns.EXP_DESCRIPTION, Column.ColumnDataType.STRING), + EXP_UNIT(ExperimentObservation.Columns.EXP_UNIT, Column.ColumnDataType.INTEGER), + EXP_TYPE(ExperimentObservation.Columns.EXP_TYPE, Column.ColumnDataType.STRING), + ENV(ExperimentObservation.Columns.ENV, Column.ColumnDataType.STRING), + ENV_LOCATION(ExperimentObservation.Columns.ENV_LOCATION, Column.ColumnDataType.STRING), + ENV_YEAR(ExperimentObservation.Columns.ENV_YEAR, Column.ColumnDataType.INTEGER), + EXP_UNIT_ID(ExperimentObservation.Columns.EXP_UNIT_ID, Column.ColumnDataType.STRING), + REP_NUM(ExperimentObservation.Columns.REP_NUM, Column.ColumnDataType.INTEGER), + BLOCK_NUM(ExperimentObservation.Columns.BLOCK_NUM, Column.ColumnDataType.INTEGER), + ROW(ExperimentObservation.Columns.ROW, Column.ColumnDataType.INTEGER), + COLUMN(ExperimentObservation.Columns.COLUMN, Column.ColumnDataType.INTEGER), + TREATMENT_FACTORS(ExperimentObservation.Columns.TREATMENT_FACTORS, Column.ColumnDataType.STRING), + OBS_UNIT_ID(ExperimentObservation.Columns.OBS_UNIT_ID, Column.ColumnDataType.STRING); + + private final Column column; + + ExperimentFileColumns(String value, Column.ColumnDataType dataType) { + this.column = new Column(value, dataType); + } + + @Override + public String toString() { + return column.getValue(); + } + + public static List getOrderedColumns() { + return Arrays.stream(ExperimentFileColumns.values()) + .map(value -> value.column) + .collect(Collectors.toList()); + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java b/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java index dc77c0479..9798379fb 100644 --- a/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java +++ b/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java @@ -19,7 +19,9 @@ import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; import io.micronaut.http.exceptions.HttpStatusException; +import io.micronaut.http.server.types.files.StreamedFile; import org.apache.commons.lang3.tuple.Pair; import org.breedinginsight.api.model.v1.request.query.PaginationParams; import org.breedinginsight.api.model.v1.request.query.QueryParams; @@ -86,6 +88,16 @@ public static HttpResponse> getSingleResponse(Object data) { return HttpResponse.ok(new Response(metadata, data)); } + // File download + public static HttpResponse getExportResponse() { + return processExportResponse(); + } + + private static HttpResponse processExportResponse() { + HttpResponse response = HttpResponse.status(HttpStatus.OK).contentType(MediaType.forFilename("download.xlsx")); + return response; + } + private static HttpResponse>> processSearchResponse( List data, SearchRequest searchRequest, QueryParams queryParams, AbstractQueryMapper mapper, Metadata metadata) { diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index ea9c7b74d..68d997073 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -1,38 +1,68 @@ package org.breedinginsight.brapi.v2; import com.google.gson.*; +import com.ibm.icu.text.UFormat; import io.kowalski.fannypack.FannyPack; +import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; import io.micronaut.http.client.RxHttpClient; import io.micronaut.http.client.annotation.Client; import io.micronaut.http.client.exceptions.HttpClientResponseException; +import io.micronaut.http.client.netty.FullNettyClientHttpResponse; import io.micronaut.http.netty.cookies.NettyCookie; import io.micronaut.test.annotation.MicronautTest; import io.reactivex.Flowable; +import lombok.SneakyThrows; +import org.apache.poi.hssf.usermodel.HSSFSheet; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.apache.poi.xssf.usermodel.XSSFWorkbookFactory; +import org.apache.xmlbeans.impl.xb.ltgfmt.TestsDocument; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.BrAPITest; import org.breedinginsight.TestUtils; +import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.model.v1.request.ProgramRequest; import org.breedinginsight.api.model.v1.request.SharedOntologyProgramRequest; import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.api.v1.controller.TestTokenValidator; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.ImportTestUtils; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.SpeciesDAO; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.*; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.services.writers.CSVWriter; +import org.breedinginsight.utilities.FileUtil; +import org.jooq.ContentType; import org.jooq.DSLContext; +import org.jooq.tools.csv.CSVReader; import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import tech.tablesaw.api.Row; +import tech.tablesaw.api.StringColumn; +import tech.tablesaw.api.Table; import javax.inject.Inject; -import java.io.File; +import javax.validation.constraints.AssertTrue; +import java.io.*; +import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.util.*; +import java.util.stream.Collectors; import static io.micronaut.http.HttpRequest.*; import static org.junit.jupiter.api.Assertions.*; @@ -44,11 +74,17 @@ public class ExperimentControllerIntegrationTest extends BrAPITest { private FannyPack securityFp; private FannyPack brapiFp; + private FannyPack fp; private FannyPack brapiObservationFp; private Program program; - private String germplasmImportId; + private ImportTestUtils importTestUtils; + private String mappingId; + private List columns = new ArrayList<>(); + private List traits; private final String GERMPLASM_LIST_NAME = "Program Germplasm List"; private final String GERMPLASM_LIST_DESC = "Program Germplasm List"; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; @Inject private DSLContext dsl; @Inject @@ -57,6 +93,10 @@ public class ExperimentControllerIntegrationTest extends BrAPITest { private UserDAO userDAO; @Inject private SpeciesDAO speciesDAO; + @Inject + private OntologyService ontologyService; + @Inject + private BrAPIGermplasmDAO germplasmDAO; @Inject @Client("/${micronaut.bi.api.version}") @@ -68,6 +108,8 @@ public class ExperimentControllerIntegrationTest extends BrAPITest { @BeforeAll void setup() throws Exception { + fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); + importTestUtils = new ImportTestUtils(); securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); //brapiObservationFp = FannyPack.fill("src/test/resources/sql/brapi/BrAPIOntologyControllerIntegrationTest.sql"); @@ -87,442 +129,328 @@ void setup() throws Exception { // Test Program ProgramRequest programRequest = ProgramRequest.builder() .name("Test Program") + .abbreviation("Test") + .documentationUrl("localhost:8080") + .objective("To test things") .species(speciesRequest) .key("TEST") .build(); program = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); - dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId().toString()); - - // Add trait to program - addTrait(program); - - // Add germplasm to program - addGermplasm(program); - - // Add experiemnts to program - addExperiments(program); - } - - private void addExperiments(Program program) throws InterruptedException { - // Get experiment system import - Flowable> call = client.exchange( - GET("/import/mappings?importName=experimenttemplatemap").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - HttpResponse response = call.blockingFirst(); - germplasmImportId = JsonParser.parseString(response.body()).getAsJsonObject() - .getAsJsonObject("result") - .getAsJsonArray("data") - .get(0).getAsJsonObject().get("id").getAsString(); - - // Insert program germplasm - File file = new File("src/test/resources/files/experiment_controller/germplasm_import.csv"); - Map germplasmListInfo = Map.ofEntries( - Map.entry("germplasmListName", GERMPLASM_LIST_NAME), - Map.entry("germplasmListDescription", GERMPLASM_LIST_DESC) - ); - TestUtils.uploadDataFile(client, program.getId(), germplasmImportId, germplasmListInfo, file); - } + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); - private void addGermplasm(Program program) throws InterruptedException { - // Get germplasm system import + // Get experiment import map Flowable> call = client.exchange( - GET("/import/mappings?importName=germplasmtemplatemap").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + GET("/import/mappings?importName=ExperimentsTemplateMap") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class ); HttpResponse response = call.blockingFirst(); - germplasmImportId = JsonParser.parseString(response.body()).getAsJsonObject() + mappingId = JsonParser.parseString(response.body()).getAsJsonObject() .getAsJsonObject("result") .getAsJsonArray("data") .get(0).getAsJsonObject().get("id").getAsString(); - // Insert program germplasm - File file = new File("src/test/resources/files/experiment_controller/germplasm_import.csv"); - Map germplasmListInfo = Map.ofEntries( - Map.entry("germplasmListName", GERMPLASM_LIST_NAME), - Map.entry("germplasmListDescription", GERMPLASM_LIST_DESC) - ); - TestUtils.uploadDataFile(client, program.getId(), germplasmImportId, germplasmListInfo, file); - } - private void addTrait(Program program) { - - // Add a trait - Trait trait = new Trait(); - trait.setTraitDescription("trait 1 description"); - trait.setEntity("entity1"); - trait.setObservationVariableName("ObsVar1"); - trait.setProgramObservationLevel(ProgramObservationLevel.builder().name("plant").build()); - Scale scale1 = new Scale(); - scale1.setScaleName("Test Scale"); - scale1.setDataType(DataType.TEXT); - Method method1 = new Method(); - trait.setScale(scale1); - trait.setMethod(method1); - trait.setTraitClass("Pheno trait"); - trait.setAttribute("leaf length"); - trait.setMainAbbreviation("abbrev1"); - trait.setSynonyms(List.of("test1", "test2")); - trait.getMethod().setMethodClass("Estimation"); - trait.getMethod().setDescription("A method"); - - List traits = List.of(trait); - - // Call endpoint - TestUtils.insertTestTraits(gson, client, program, traits); - } - - @Test - @Order(1) - void getAllProgramsNoSharedPrograms() { - String url = String.format("/programs/%s/ontology/shared/programs", mainProgram.getId()); - Flowable> call = client.exchange( - GET(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.OK, response.getStatus()); - - JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); - JsonArray data = result.getAsJsonArray("data"); - assertEquals(3, data.size(), "Wrong number of programs returned"); - - // Check all are not shared and are inactive - for (JsonElement element: data) { - JsonObject program = element.getAsJsonObject(); - - assertFalse(program.get("shared").getAsBoolean(), "Shared should have been false"); - assertNull(program.get("accepted"), "Accepted should have been false"); - assertNull(program.get("editable"), "Editable should have been false"); + // Add traits to program + traits = createTraits(1); + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + try { + ontologyService.createTraits(program.getId(), traits, user, false); + } catch (ValidatorException e) { + System.err.println(e.getErrors()); + throw e; } - } - - @Test - @Order(2) - void addSharedPrograms() { - - String url = String.format("/programs/%s/ontology/shared/programs", mainProgram.getId()); - List requests = new ArrayList<>(); - requests.add(new SharedOntologyProgramRequest(otherProgram.getId(), otherProgram.getName())); - requests.add(new SharedOntologyProgramRequest(thirdProgram.getId(), thirdProgram.getName())); - String json = gson.toJson(requests); - - Flowable> call = client.exchange( - POST(url, json) - .contentType(MediaType.APPLICATION_JSON) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.OK, response.getStatus()); - - JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); - JsonArray data = result.getAsJsonArray("data"); - assertEquals(2, data.size(), "Wrong number of programs returned"); + // Add germplasm to program + List germplasm = createGermplasm(1); + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); + newReference.setReferenceID(program.getId().toString()); - // Check all are not shared and are inactive - for (JsonElement element: data) { - JsonObject program = element.getAsJsonObject(); + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); - assertTrue(program.get("shared").getAsBoolean(), "Shared should have been true"); - assertFalse(program.get("accepted").getAsBoolean(), "Accepted should have been false"); - assertTrue(program.get("editable").getAsBoolean(), "Editable should have been true"); - } + germplasmDAO.createBrAPIGermplasm(germplasm, program.getId(), null); } - @Test - @Order(3) - void subscribeOntologyProgramHasTraitsError() { - String url = String.format("/programs/%s/ontology/subscribe/%s", thirdProgram.getId(), mainProgram.getId()); - - Flowable> call = client.exchange( - PUT(url, "") - .contentType(MediaType.APPLICATION_JSON) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); + private File writeDataToFile(List> data, List traits) throws IOException { + File file = File.createTempFile("test", ".csv"); + + columns.add(Column.builder().value(ExperimentObservation.Columns.GERMPLASM_NAME).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(ExperimentObservation.Columns.GERMPLASM_GID).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(ExperimentObservation.Columns.TEST_CHECK).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(ExperimentObservation.Columns.EXP_TITLE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(ExperimentObservation.Columns.EXP_DESCRIPTION).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(ExperimentObservation.Columns.EXP_UNIT).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(ExperimentObservation.Columns.EXP_TYPE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(ExperimentObservation.Columns.ENV).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(ExperimentObservation.Columns.ENV_LOCATION).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(ExperimentObservation.Columns.ENV_YEAR).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(ExperimentObservation.Columns.EXP_UNIT_ID).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(ExperimentObservation.Columns.REP_NUM).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(ExperimentObservation.Columns.BLOCK_NUM).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(ExperimentObservation.Columns.ROW).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(ExperimentObservation.Columns.COLUMN).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(ExperimentObservation.Columns.TREATMENT_FACTORS).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(ExperimentObservation.Columns.OBS_UNIT_ID).dataType(Column.ColumnDataType.STRING).build()); + + if(traits != null) { + traits.forEach(trait -> { + columns.add(Column.builder().value(trait.getObservationVariableName()).dataType(Column.ColumnDataType.STRING).build()); + }); + } - HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { - HttpResponse response = call.blockingFirst(); - }); + ByteArrayOutputStream byteArrayOutputStream = CSVWriter.writeToCSV(columns, data); + FileOutputStream fos = new FileOutputStream(file); + fos.write(byteArrayOutputStream.toByteArray()); - assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, e.getStatus()); + return file; } - @Test - @Order(3) - void subscribeOntologySuccess() { - String url = String.format("/programs/%s/ontology/subscribe/%s", otherProgram.getId(), mainProgram.getId()); - - Flowable> call = client.exchange( - PUT(url, "") - .contentType(MediaType.APPLICATION_JSON) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.OK, response.getStatus()); + private Map makeExpImportRow(String environment) { + Map row = new HashMap<>(); + row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); + row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); + row.put(ExperimentObservation.Columns.EXP_TITLE, "Test Exp"); + row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); + row.put(ExperimentObservation.Columns.ENV, environment); + row.put(ExperimentObservation.Columns.ENV_LOCATION, "Location A"); + row.put(ExperimentObservation.Columns.ENV_YEAR, "2023"); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, "a-1"); + row.put(ExperimentObservation.Columns.REP_NUM, "1"); + row.put(ExperimentObservation.Columns.BLOCK_NUM, "1"); + row.put(ExperimentObservation.Columns.ROW, "1"); + row.put(ExperimentObservation.Columns.COLUMN, "1"); + return row; } - @Test - @Order(4) - void getSubscribedOntologyOptions() { - - String url = String.format("/programs/%s/ontology/subscribe", otherProgram.getId()); - Flowable> call = client.exchange( - GET(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.OK, response.getStatus()); - - JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); - JsonArray data = result.getAsJsonArray("data"); - assertEquals(1, data.size(), "Wrong number of programs returned"); - - // Check all are not shared and are inactive - for (JsonElement element: data) { - JsonObject program = element.getAsJsonObject(); - - assertTrue(program.get("subscribed").getAsBoolean()); - assertTrue(program.get("editable").getAsBoolean()); + private List createTraits(int numToCreate) { + List traits = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String varName = "tt_test_" + (i + 1); + traits.add(Trait.builder() + .observationVariableName(varName) + .entity("Plant " + i) + .attribute("height " + i) + .traitDescription("test") + .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build()) + .scale(Scale.builder() + .scaleName("test scale") + .dataType(DataType.NUMERICAL) + .validValueMin(0) + .validValueMax(100) + .build()) + .method(Method.builder() + .description("test method") + .methodClass("test method") + .build()) + .build()); } - } - - @Test - @Order(4) - void getAllProgramsSharedPrograms() { - String url = String.format("/programs/%s/ontology/shared/programs", mainProgram.getId()); - Flowable> call = client.exchange( - GET(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.OK, response.getStatus()); - - JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); - JsonArray data = result.getAsJsonArray("data"); - assertEquals(3, data.size(), "Wrong number of programs returned"); - - // Check all are not shared and are inactive - for (JsonElement element: data) { - JsonObject program = element.getAsJsonObject(); - - if (program.get("programId").getAsString().equals(fourthProgram.getId().toString())) { - assertFalse(program.get("shared").getAsBoolean()); - } else { - assertTrue(program.get("shared").getAsBoolean(), "Shared should have been true"); - assertTrue(program.get("editable").getAsBoolean(), "Editable should have been false"); - if (program.get("programId").getAsString().equals(otherProgram.getId().toString())) { - assertTrue(program.get("accepted").getAsBoolean()); - } else { - assertFalse(program.get("accepted").getAsBoolean()); - } - } - } + return traits; } - @Test - @Order(4) - void getOnlySharedPrograms() { - String url = String.format("/programs/%s/ontology/shared/programs?shared=true", mainProgram.getId()); - Flowable> call = client.exchange( - GET(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.OK, response.getStatus()); - - JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); - JsonArray data = result.getAsJsonArray("data"); - assertEquals(2, data.size(), "Wrong number of programs returned"); - - // Check all are not shared and are inactive - for (JsonElement element: data) { - JsonObject program = element.getAsJsonObject(); - - assertTrue(program.get("shared").getAsBoolean(), "Shared should have been true"); + private List createGermplasm(int numToCreate) { + List germplasm = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String gid = ""+(i+1); + BrAPIGermplasm testGermplasm = new BrAPIGermplasm(); + testGermplasm.setGermplasmName(String.format("Germplasm %s [TEST-%s]", gid, gid)); + testGermplasm.setSeedSource("Wild"); + testGermplasm.setAccessionNumber(gid); + testGermplasm.setDefaultDisplayName(String.format("Germplasm %s", gid)); + JsonObject additionalInfo = new JsonObject(); + additionalInfo.addProperty("importEntryNumber", gid); + additionalInfo.addProperty("breedingMethod", "Allopolyploid"); + testGermplasm.setAdditionalInfo(additionalInfo); + List externalRef = new ArrayList<>(); + BrAPIExternalReference testReference = new BrAPIExternalReference(); + testReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); + testReference.setReferenceID(UUID.randomUUID().toString()); + externalRef.add(testReference); + testGermplasm.setExternalReferences(externalRef); + germplasm.add(testGermplasm); } - } - - @Test - @Order(4) - void shareOntologySubscribedToOtherProgram() { - // Cannot share ontology if you are using a shared ontology - String url = String.format("/programs/%s/ontology/shared/programs", otherProgram.getId()); - List requests = new ArrayList<>(); - requests.add(new SharedOntologyProgramRequest(fourthProgram.getId(), fourthProgram.getName())); - String json = gson.toJson(requests); - - Flowable> call = client.exchange( - POST(url, json) - .contentType(MediaType.APPLICATION_JSON) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { - HttpResponse response = call.blockingFirst(); - }); - assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, e.getStatus()); + return germplasm; } - @Test - @Order(4) - void revokeOntologyUneditable() { - // Ontology cannot be revoke if shared program has accepted and has observations - super.getBrapiDsl().execute(brapiObservationFp.get("AddObservations"), otherProgram.getId().toString()); - - String url = String.format("/programs/%s/ontology/shared/programs/%s", mainProgram.getId(), otherProgram.getId()); - Flowable> call = client.exchange( - DELETE(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); + private void checkXLSDownload(Map dataRow, Map emptyRow, HSSFWorkbook workbook) { + HSSFSheet experimentData = workbook.getSheet("Experiment Data"); + // Filename is correct: _Observation Dataset [-]__ + // All columns included + // All environments included + // All data included + // Observation units populated - HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { - HttpResponse response = call.blockingFirst(); - }); - assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, e.getStatus()); - - super.getBrapiDsl().execute(brapiObservationFp.get("DeleteObservations")); } - @Test - @Order(4) - void unsubscribeOntologyUneditableError() { - super.getBrapiDsl().execute(brapiObservationFp.get("AddObservations"), otherProgram.getId().toString()); - - String url = String.format("/programs/%s/ontology/subscribe/%s", otherProgram.getId(), mainProgram.getId()); - Flowable> call = client.exchange( - DELETE(url, "").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - - HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { - HttpResponse response = call.blockingFirst(); - }); - assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, e.getStatus()); + private void checkXLSXDownload(List> importRows, XSSFWorkbook workbook) { + // Filename is correct: _Observation Dataset [-]__ + // All columns included + // All environments included + // All data included + // Observation units populated - super.getBrapiDsl().execute(brapiObservationFp.get("DeleteObservations")); } - @Test - @Order(5) - void unsubscribeOntologySuccess() { + private void checkCSVDownload(List> importRows, CSVReader reader) { + // Filename is correct: _Observation Dataset [-]__ + // All columns included + // All environments included + // All data included + // Observation units populated - String url = String.format("/programs/%s/ontology/subscribe/%s", otherProgram.getId(), mainProgram.getId()); - Flowable> call = client.exchange( - DELETE(url, "").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.OK, response.getStatus()); } + private void checkDownloadTable(String requestedEnv, List> importRows, Table table) { + // Filename is correct: _Observation Dataset [-]__ + List> requestedImportRows; - @Test - @Order(5) - void revokeOntologySuccess() { + // All columns included + assertEquals(columns.size(), table.columnCount()); - String url = String.format("/programs/%s/ontology/shared/programs/%s", mainProgram.getId(), thirdProgram.getId()); - Flowable> call = client.exchange( - DELETE(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); + if (requestedEnv == null) { + requestedImportRows = importRows; - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.OK, response.getStatus()); - } + // All environments downloaded + importRows.stream() + .map(row -> row.get(ExperimentObservation.Columns.ENV).toString()) + .distinct() + .collect(Collectors.toList()) + .forEach(envName -> { + assertTrue(table.stringColumn("Env").contains(envName)); + }); - @Test - void shareSelfError() { - // Test that program cannot share with themselves + } else { - String url = String.format("/programs/%s/ontology/shared/programs", mainProgram.getId()); - List requests = new ArrayList<>(); - requests.add(new SharedOntologyProgramRequest(mainProgram.getId(), mainProgram.getName())); - String json = gson.toJson(requests); + // Only requested environment downloaded + requestedImportRows = importRows.stream().filter(row -> { + return row.get("Env").toString().equals(requestedEnv); + }).collect(Collectors.toList()); - Flowable> call = client.exchange( - POST(url, json) - .contentType(MediaType.APPLICATION_JSON) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); + assertEquals(1, table.stringColumn("Env").countUnique()); + assertTrue(table.stringColumn("Env").contains(requestedEnv)); - HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { - HttpResponse response = call.blockingFirst(); - }); - assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, e.getStatus()); - } + } - @Test - void revokeOntologyProgramNotExist() { - String url = String.format("/programs/%s/ontology/shared/programs/%s", UUID.randomUUID(), otherProgram.getId()); - Flowable> call = client.exchange( - DELETE(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); + // All requested import data included in download + List> matchingImportRows = requestedImportRows.stream().filter(importRow -> { + for (Row downloadRow : table) { + if (isMatchedRow(importRow, downloadRow)) { + return true; + }; + } + return false; + }).collect(Collectors.toList()); + assertEquals(requestedImportRows.size(),matchingImportRows.size()); - HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { - HttpResponse response = call.blockingFirst(); - }); - assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + // Observation units populated + assertEquals(0, table.column("ObsUnitID").countMissing()); + assertEquals(importRows.size(), table.column("ObsUnitID").countUnique()); } - @Test - void revokeOntologySharedProgramNotExist() { - String url = String.format("/programs/%s/ontology/shared/programs/%s", mainProgram.getId(), UUID.randomUUID()); - Flowable> call = client.exchange( - DELETE(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - - HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { - HttpResponse response = call.blockingFirst(); - }); - assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); - } + private boolean isMatchedRow(Map importRow, Row downloadRow) { + return importRow.entrySet().stream().filter(e -> { + String header = e.getKey(); + List importColumns = columns.stream().filter(col -> {return col.getValue() == header;}).collect(Collectors.toList()); + if (importColumns.isEmpty() || importColumns.size() > 1) { + return false; + } - @Test - void revokeOntologySharedProgramNotShared() { - String url = String.format("/programs/%s/ontology/shared/programs/%s", mainProgram.getId(), fourthProgram.getId()); - Flowable> call = client.exchange( - DELETE(url).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); + if (importColumns.get(0).getDataType() == Column.ColumnDataType.STRING) { + return downloadRow.getString(e.getKey().toString()) == e.getValue(); + } - HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { - HttpResponse response = call.blockingFirst(); - }); - assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + if (importColumns.get(0).getDataType() == Column.ColumnDataType.INTEGER) { + return downloadRow.getInt(e.getKey().toString()) == Integer.parseInt(e.getValue().toString()) ; + } + return false; + }).collect(Collectors.toList()).size() == importRow.size(); } - @Test - void shareOntologyProgramNotExist() { - - String url = String.format("/programs/%s/ontology/shared/programs", mainProgram.getId()); - List requests = new ArrayList<>(); - requests.add(new SharedOntologyProgramRequest(UUID.randomUUID(), "I don't exist")); - String json = gson.toJson(requests); - - Flowable> call = client.exchange( - POST(url, json) - .contentType(MediaType.APPLICATION_JSON) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); + /* + Tests + - export empty dataset, single environment, csv format + - export empty dataset, single environment, xls format + - export empty dataset, single environment, xlsx format + - export populated dataset, single environment, csv format + - export populated dataset, single environment, xls format + - export populated dataset, single environment, xlsx format + - export empty dataset, multiple environment, csv format + - export empty dataset, multiple environment, xls format + - export empty dataset, multiple environment, xlsx format + - export populated dataset, multiple environment, csv format + - export populated dataset, multiple environment, xls format + - export populated dataset, multiple environment, xlsx format + */ + + @ParameterizedTest + @CsvSource(value = {"true,,XLSX", "false,,CSV", "true,Env1,CSV", "false,Env1,CSV", + "true,,XLS", "false,,XLS", "true,Env1,XLS", "false,Env1,XLS", + "true,,XLSX", "false,,XLSX", "true,Env1,XLSX", "false,Env1,XLSX",}) + @SneakyThrows + void downloadDatasets(boolean hasEmptyObs, String requestedEnv, String extension) { + // Make test experiment import + List> rows = new ArrayList<>(); + Map row1 = makeExpImportRow("Env1"); + Map row2 = makeExpImportRow("Env2"); + + // Add test observation data + for (int i = 0; i < traits.size(); i++) { + row1.put(traits.get(i).getObservationVariableName(), Integer.toString(i)); + if (!hasEmptyObs) { + row2.put(traits.get(i).getObservationVariableName(), Integer.toString(i)); + } + }; - HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { - HttpResponse response = call.blockingFirst(); - }); - assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, e.getStatus()); - } + rows.add(row1); + rows.add(row2); - @Test - void shareOntologySharedProgramNotExist() { + // Import test experiment, environments, and any observations + JsonObject importResult = importTestUtils.uploadAndFetch(writeDataToFile(rows, traits), null, true, client, program, mappingId); + String experimentId = importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); - String url = String.format("/programs/%s/ontology/shared/programs", UUID.randomUUID()); - List requests = new ArrayList<>(); - requests.add(new SharedOntologyProgramRequest(otherProgram.getId(), otherProgram.getName())); - String json = gson.toJson(requests); - Flowable> call = client.exchange( - POST(url, json) - .contentType(MediaType.APPLICATION_JSON) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + // Download test experiment + String envParam = "all=true"; + if (requestedEnv != null) { + envParam = "env=" + requestedEnv; + } + Flowable> call = client.exchange( + GET(String.format("/programs/%s/experiments/%s/export?%s&fileExtension=%s",program.getId().toString(), experimentId, envParam, extension)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class ); + HttpResponse response = call.blockingFirst(); - HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { - HttpResponse response = call.blockingFirst(); - }); - assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + // Assert 200 response + assertEquals(HttpStatus.OK, response.getStatus()); + + // Assert file format fidelity + Map mediaTypeByExtension = new HashMap<>(); + mediaTypeByExtension.put("CSV", "text/csv"); + mediaTypeByExtension.put("XLS","application/vnd.ms.excel"); + mediaTypeByExtension.put("XLSX", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + String downloadMediaType = response.getHeaders().getContentType().orElseThrow(Exception::new); + assertEquals(mediaTypeByExtension.get(extension), downloadMediaType); + + // Assert import/export fidelity and presence of observation units in export + ByteArrayInputStream bodyStream = new ByteArrayInputStream(response.body()); + Table download = Table.create(); + if (extension.equals("CSV")) { + download = FileUtil.parseTableFromCsv(bodyStream); + } + if (extension.equals("XLS") || extension.equals("XLSX")) { + Workbook book = WorkbookFactory.create(bodyStream); + XSSFWorkbook workbook = new XSSFWorkbook(bodyStream); + //new XSSFWorkbook(new ByteArrayInputStream(((FullNettyClientHttpResponse) response).bodyBytes)) + download = FileUtil.parseTableFromExcel(bodyStream, 0); + } + checkDownloadTable(requestedEnv, rows, download); } } diff --git a/src/test/resources/sql/ImportControllerIntegrationTest.sql b/src/test/resources/sql/ImportControllerIntegrationTest.sql index 42da3cd41..8a93b472e 100644 --- a/src/test/resources/sql/ImportControllerIntegrationTest.sql +++ b/src/test/resources/sql/ImportControllerIntegrationTest.sql @@ -30,6 +30,12 @@ INSERT INTO public.importer_mapping (name,import_type_id,mapping,file,draft,crea '[{"Name": "Chris-BB_germ_1", "Breeding Method": "BCR", "Source": "Wild"}, {"Name": "Chris-BB_germ_2", "Breeding Method": "BCR", "Source": "Wild"}, {"Name": "Chris-BB_germ_3", "Breeding Method": "BCR", "Source": "Wild"}, {"Name": "Chris-BB_germ_4", "Breeding Method": "BCR", "Source": "Wild"}]', false,user_id,user_id); +INSERT INTO public.importer_mapping (name,import_type_id,mapping,file,draft,created_by,updated_by) values + ('ExperimentsTemplateMap','ExperimentImport', + '[{"id": "726a9f10-4892-4204-9e52-bd2b1d735f65", "value": {"fileFieldName": "Germplasm Name"}, "objectId": "germplasmName"}, {"id": "98774e20-6f89-4d6a-a7c9-f88887228ed6", "value": {"fileFieldName": "Germplasm GID"}, "objectId": "gid"}, {"id": "880ef0c9-4e3e-42d4-9edc-667684a91889", "value": {"fileFieldName": "Test (T) or Check (C )"}, "objectId": "test_or_check"}, {"id": "b693eca7-efcd-4518-a9d3-db0b037a76ee", "value": {"fileFieldName": "Exp Title"}, "objectId": "exp_title"}, {"id": "df340215-db6e-4219-a3b7-119f297b81c3", "value": {"fileFieldName": "Exp Description"}, "objectId": "expDescription"}, {"id": "9ca7cc81-562c-43a7-989a-41da309f603d", "value": {"fileFieldName": "Exp Unit"}, "objectId": "expUnit"}, {"id": "27215777-c8f9-4fe7-a7ac-918d6168b0dd", "value": {"fileFieldName": "Exp Type"}, "objectId": "expType"}, {"id": "19d220e2-dff0-4a3a-ad6e-32f4d8602b5c", "value": {"fileFieldName": "Env"}, "objectId": "env"}, {"id": "861518b9-5c9e-4fe5-b31e-baf16e27155d", "value": {"fileFieldName": "Env Location"}, "objectId": "envLocation"}, {"id": "667355c3-dae1-4a64-94c8-ac2d543bd474", "value": {"fileFieldName": "Env Year"}, "objectId": "envYear"}, {"id": "ad11f2df-c5b4-4a05-8e52-c57625140061", "value": {"fileFieldName": "Exp Unit ID"}, "objectId": "expUnitId"}, {"id": "639b40ec-20f8-4659-8464-6a4be997ac7a", "value": {"fileFieldName": "Exp Replicate #"}, "objectId": "expReplicateNo"}, {"id": "2a62a80f-d8ba-42c4-9997-3b4ac8a965aa", "value": {"fileFieldName": "Exp Block #"}, "objectId": "expBlockNo"}, {"id": "f3e7de69-21ad-4cda-b1cc-a5e1987fb931", "value": {"fileFieldName": "Row"}, "objectId": "row"}, {"id": "251c5bbd-fc4d-4371-a4ce-4e2686f6837e", "value": {"fileFieldName": "Column"}, "objectId": "column"}, {"id": "ce5f61f2-f1de-45a4-8baf-e2471a5d863d", "value": {"fileFieldName": "Treatment Factors"}, "objectId": "treatmentFactors"}, {"id": "c5b8276f-e777-4385-a80f-5199abe63aac", "value": {"fileFieldName": "ObsUnitID"}, "objectId": "ObsUnitID"}]', + '[{"Env": "New Study", "Row": 4, "Column": 5, "Env Year": 2012, "Exp Type": "phenotyping", "Exp Unit": "plot", "Exp Title": "New Trial", "ObsUnitID": "", "Exp Block #": 2, "Exp Unit ID": 3, "Env Location": "New Location", "Germplasm GID": 1, "Germplasm Name": "Test", "Exp Description": "A new trial", "Exp Replicate #": 0, "Treatment Factors": "Jam application", "Test (T) or Check (C )": true}]', + false,user_id,user_id); + END $$; From 5c199850458c6dc32445620e1301327e3f7c34cc Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 17 May 2023 20:55:00 -0400 Subject: [PATCH 03/27] add BrAPITrialService method --- .../model/v1/request/query/DatasetParams.java | 49 ++++++++++++ .../v1/request/query/FileDownloadParams.java | 22 ++++++ .../brapi/v2/services/BrAPITrialService.java | 25 +++++- .../importer/daos/BrAPIObservationDAO.java | 16 ++++ .../brapps/importer/daos/BrAPITrialDAO.java | 19 +++++ .../breedinginsight/utilities/FileUtil.java | 2 + .../ExperimentControllerIntegrationTest.java | 78 ++++++++++++------- 7 files changed, 181 insertions(+), 30 deletions(-) create mode 100644 src/main/java/org/breedinginsight/api/model/v1/request/query/DatasetParams.java create mode 100644 src/main/java/org/breedinginsight/api/model/v1/request/query/FileDownloadParams.java diff --git a/src/main/java/org/breedinginsight/api/model/v1/request/query/DatasetParams.java b/src/main/java/org/breedinginsight/api/model/v1/request/query/DatasetParams.java new file mode 100644 index 000000000..fa5929d71 --- /dev/null +++ b/src/main/java/org/breedinginsight/api/model/v1/request/query/DatasetParams.java @@ -0,0 +1,49 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.model.v1.request.query; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import io.micronaut.core.annotation.Introspected; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.breedinginsight.api.v1.controller.metadata.SortOrder; +import org.breedinginsight.brapps.importer.model.exports.FileType; + +import javax.validation.constraints.Positive; + + +@Getter +@Setter +@Introspected +@NoArgsConstructor +public class DatasetParams implements FileDownloadParams { + + private static String DEFAULT_FILETYPE_NAME = FileType.XLSX.getName(); + + private String fileExtension; + private String dataset; + private String environments; + @JsonIgnore + private boolean includeTimestamps; + + public String getDefaultFileTypeName() { + return DEFAULT_FILETYPE_NAME; + } + +} diff --git a/src/main/java/org/breedinginsight/api/model/v1/request/query/FileDownloadParams.java b/src/main/java/org/breedinginsight/api/model/v1/request/query/FileDownloadParams.java new file mode 100644 index 000000000..622aa3143 --- /dev/null +++ b/src/main/java/org/breedinginsight/api/model/v1/request/query/FileDownloadParams.java @@ -0,0 +1,22 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.api.model.v1.request.query; + +public interface FileDownloadParams { + + String getDefaultFileTypeName(); +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index ed9de26a3..2d6cdcb33 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -11,6 +11,7 @@ import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.Utilities; import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapps.importer.daos.BrAPIObservationDAO; import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; @@ -21,6 +22,7 @@ import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; import org.breedinginsight.services.writers.CSVWriter; import org.breedinginsight.services.writers.ExcelWriter; +import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; @@ -33,11 +35,16 @@ public class BrAPITrialService { private final BrAPITrialDAO trialDAO; private final BrAPIObservationUnitDAO ouDAO; + private final BrAPIObservationDAO observationDAO; + private final ProgramService programService; @Inject - public BrAPITrialService( BrAPITrialDAO trialDAO,BrAPIObservationUnitDAO ouDAO) { + public BrAPITrialService(ProgramService programService, BrAPITrialDAO trialDAO, BrAPIObservationUnitDAO ouDAO, BrAPIObservationDAO observationDAO) { + this.programService = programService; this.trialDAO = trialDAO; this.ouDAO = ouDAO; + this.observationDAO = observationDAO; + } public List getExperiments(UUID programId) throws ApiException, DoesNotExistException { @@ -105,8 +112,22 @@ private Map makeExpObsMapByHeader(ExperimentObservation obs) { return row; } - public List getDataset() { + public List getDataset(UUID programId, UUID experimentId, UUID datasetId) throws ApiException, DoesNotExistException { List dataset = new ArrayList<>(); + try { + List trials = trialDAO.getTrialsByExperimentIds(List.of(experimentId), programId); + if (trials.isEmpty()) { + throw new DoesNotExistException("Trial does not exist"); + } + + dataset = observationDAO.getObservationsByTrialDbId(List.of(experimentId.toString()), programId); + } catch (ApiException e) { + log.error("Error fetching BrAPI observations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } catch (DoesNotExistException e) { + log.error("Trial does not exist", e); + throw new DoesNotExistException(e.getMessage()); + } return dataset; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java index ecaa4bff0..6085341fe 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java @@ -69,6 +69,22 @@ public List getObservationsByStudyName(List studyNames ); } + public List getObservationsByTrialDbId(List trialDbIds, UUID programId) throws ApiException { + if(trialDbIds.isEmpty()) { + return Collections.emptyList(); + } + + BrAPIObservationSearchRequest observationSearchRequest = new BrAPIObservationSearchRequest(); + observationSearchRequest.setProgramDbIds(List.of(programId.toString())); + observationSearchRequest.setTrialDbIds(new ArrayList<>(trialDbIds)); + ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); + return brAPIDAOUtil.search( + api::searchObservationsPost, + (brAPIWSMIMEDataTypes, searchResultsDbId, page, pageSize) -> searchObservationsSearchResultsDbIdGet(programId, searchResultsDbId, page, pageSize), + observationSearchRequest + ); + } + public List getObservationsByObservationUnitsAndVariables(Collection ouDbIds, Collection variableDbIds, Program program) throws ApiException { if(ouDbIds.isEmpty() || variableDbIds.isEmpty()) { return Collections.emptyList(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index ab9de443c..7a2d2485c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -34,6 +34,7 @@ import javax.inject.Inject; import javax.inject.Singleton; import java.util.*; +import java.util.stream.Collectors; @Singleton public class BrAPITrialDAO { @@ -162,6 +163,24 @@ public List getTrialsByDbIds(Collection trialDbIds, Program trialSearch ); } + + public List getTrialsByExperimentIds(Collection trialExperimentIds, UUID programId) throws ApiException { + if(trialExperimentIds.isEmpty()) { + return Collections.emptyList(); + } + + BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); + trialSearch.programDbIds(List.of(programId.toString())); + trialSearch.externalReferenceSources(List.of(ExternalReferenceSource.TRIALS.getName())); + trialSearch.externalReferenceIDs(trialExperimentIds.stream().map(id -> id.toString()).collect(Collectors.toList())); + TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); + return brAPIDAOUtil.search( + api::searchTrialsPost, + api::searchTrialsSearchResultsDbIdGet, + trialSearch + ); + } + } diff --git a/src/main/java/org/breedinginsight/utilities/FileUtil.java b/src/main/java/org/breedinginsight/utilities/FileUtil.java index 37f401f76..085115973 100644 --- a/src/main/java/org/breedinginsight/utilities/FileUtil.java +++ b/src/main/java/org/breedinginsight/utilities/FileUtil.java @@ -40,6 +40,7 @@ public class FileUtil { // For backward compatibility private static final String OLD_GERMPLASM_EXCEL_DATA_SHEET_NAME = "Germplasm Import"; private static final String OLD_EXPERIMENT_EXCEL_DATA_SHEET_NAME = "Experiment Data"; + private static final String DATASET_EXPORT_EXCEL_DATA_SHEET_NAME = "Dataset Export"; public static Table parseTableFromExcel(InputStream inputStream, Integer headerRowIndex) throws ParsingException { @@ -56,6 +57,7 @@ public static Table parseTableFromExcel(InputStream inputStream, Integer headerR //For backward compatibility allow old sheet names if( sheet == null){ sheet = workbook.getSheet(OLD_GERMPLASM_EXCEL_DATA_SHEET_NAME); } if( sheet == null){ sheet = workbook.getSheet(OLD_EXPERIMENT_EXCEL_DATA_SHEET_NAME); } + if( sheet == null){ sheet = workbook.getSheet(DATASET_EXPORT_EXCEL_DATA_SHEET_NAME); } if (sheet == null) { throw new ParsingException(ParsingExceptionType.MISSING_SHEET); diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index 68d997073..67b8c5d9c 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -79,6 +79,8 @@ public class ExperimentControllerIntegrationTest extends BrAPITest { private Program program; private ImportTestUtils importTestUtils; private String mappingId; + private String experimentId; + private List> rows = new ArrayList<>(); private List columns = new ArrayList<>(); private List traits; private final String GERMPLASM_LIST_NAME = "Program Germplasm List"; @@ -171,6 +173,29 @@ void setup() throws Exception { germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); germplasmDAO.createBrAPIGermplasm(germplasm, program.getId(), null); + + // Make test experiment import + Map row1 = makeExpImportRow("Env1"); + Map row2 = makeExpImportRow("Env2"); + + // Add test observation data + for (int i = 0; i < traits.size(); i++) { + row1.put(traits.get(i).getObservationVariableName(), Integer.toString(i)); + row2.put(traits.get(i).getObservationVariableName(), Integer.toString(i)); + }; + + rows.add(row1); + rows.add(row2); + + // Import test experiment, environments, and any observations + JsonObject importResult = importTestUtils.uploadAndFetch(writeDataToFile(rows, traits), null, true, client, program, mappingId); + experimentId = importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + } private File writeDataToFile(List> data, List traits) throws IOException { @@ -386,35 +411,35 @@ private boolean isMatchedRow(Map importRow, Row downloadRow) { */ @ParameterizedTest - @CsvSource(value = {"true,,XLSX", "false,,CSV", "true,Env1,CSV", "false,Env1,CSV", + @CsvSource(value = {"true,,CSV", "false,,CSV", "true,Env1,CSV", "false,Env1,CSV", "true,,XLS", "false,,XLS", "true,Env1,XLS", "false,Env1,XLS", "true,,XLSX", "false,,XLSX", "true,Env1,XLSX", "false,Env1,XLSX",}) @SneakyThrows void downloadDatasets(boolean hasEmptyObs, String requestedEnv, String extension) { - // Make test experiment import - List> rows = new ArrayList<>(); - Map row1 = makeExpImportRow("Env1"); - Map row2 = makeExpImportRow("Env2"); - - // Add test observation data - for (int i = 0; i < traits.size(); i++) { - row1.put(traits.get(i).getObservationVariableName(), Integer.toString(i)); - if (!hasEmptyObs) { - row2.put(traits.get(i).getObservationVariableName(), Integer.toString(i)); - } - }; - - rows.add(row1); - rows.add(row2); - - // Import test experiment, environments, and any observations - JsonObject importResult = importTestUtils.uploadAndFetch(writeDataToFile(rows, traits), null, true, client, program, mappingId); - String experimentId = importResult - .get("preview").getAsJsonObject() - .get("rows").getAsJsonArray() - .get(0).getAsJsonObject() - .get("trial").getAsJsonObject() - .get("id").getAsString(); +// // Make test experiment import +// List> rows = new ArrayList<>(); +// Map row1 = makeExpImportRow("Env1"); +// Map row2 = makeExpImportRow("Env2"); +// +// // Add test observation data +// for (int i = 0; i < traits.size(); i++) { +// row1.put(traits.get(i).getObservationVariableName(), Integer.toString(i)); +// if (!hasEmptyObs) { +// row2.put(traits.get(i).getObservationVariableName(), Integer.toString(i)); +// } +// }; +// +// rows.add(row1); +// rows.add(row2); +// +// // Import test experiment, environments, and any observations +// JsonObject importResult = importTestUtils.uploadAndFetch(writeDataToFile(rows, traits), null, true, client, program, mappingId); +// String experimentId = importResult +// .get("preview").getAsJsonObject() +// .get("rows").getAsJsonArray() +// .get(0).getAsJsonObject() +// .get("trial").getAsJsonObject() +// .get("id").getAsString(); // Download test experiment @@ -446,9 +471,6 @@ void downloadDatasets(boolean hasEmptyObs, String requestedEnv, String extension download = FileUtil.parseTableFromCsv(bodyStream); } if (extension.equals("XLS") || extension.equals("XLSX")) { - Workbook book = WorkbookFactory.create(bodyStream); - XSSFWorkbook workbook = new XSSFWorkbook(bodyStream); - //new XSSFWorkbook(new ByteArrayInputStream(((FullNettyClientHttpResponse) response).bodyBytes)) download = FileUtil.parseTableFromExcel(bodyStream, 0); } checkDownloadTable(requestedEnv, rows, download); From f8afcfc5d7661c7d54baa4d990ce841a8508ceb4 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 18 May 2023 15:15:30 -0400 Subject: [PATCH 04/27] check for datsetId in BrAPITrialService#getDataset --- .../brapi/v2/services/BrAPITrialService.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 2d6cdcb33..7a33f1aaf 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -11,8 +11,10 @@ import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.Utilities; import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.daos.BrAPIObservationDAO; import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.model.base.AdditionalInfo; import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.model.Column; @@ -28,6 +30,7 @@ import javax.inject.Singleton; import java.io.IOException; import java.util.*; +import java.util.stream.Collectors; @Slf4j @Singleton @@ -117,10 +120,15 @@ public List getDataset(UUID programId, UUID experimentId, UUID try { List trials = trialDAO.getTrialsByExperimentIds(List.of(experimentId), programId); if (trials.isEmpty()) { - throw new DoesNotExistException("Trial does not exist"); + throw new DoesNotExistException("Experiment not found"); } - - dataset = observationDAO.getObservationsByTrialDbId(List.of(experimentId.toString()), programId); + BrAPITrial datasetTrial = trials.stream().filter(trial -> { + String id = trial + .getAdditionalInfo() + .getAsJsonObject(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); + return id != null; + }).findAny().orElseThrow(() -> new DoesNotExistException("Experiment dataset not found")); + dataset = observationDAO.getObservationsByTrialDbId(List.of(datasetTrial.getTrialDbId()), programId); } catch (ApiException e) { log.error("Error fetching BrAPI observations: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); From 19de7d963801aa0872a918f5db28220c4346bb4f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 22 May 2023 10:41:03 -0400 Subject: [PATCH 05/27] add obsvar columns --- .../brapi/v2/ExperimentController.java | 22 ++- .../request/query/ExperimentExportQuery.java | 21 +++ .../brapi/v2/services/BrAPITrialService.java | 138 ++++++++++++++++-- .../importer/daos/BrAPIObservationDAO.java | 8 +- .../brapps/importer/daos/BrAPITrialDAO.java | 13 +- .../ExperimentControllerIntegrationTest.java | 4 +- 6 files changed, 181 insertions(+), 25 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index 5615bff8a..508a3d9a2 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -11,6 +11,8 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationVariable; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.api.model.v1.request.query.SearchRequest; @@ -18,9 +20,14 @@ import org.breedinginsight.api.model.v1.response.Response; import org.breedinginsight.api.model.v1.validators.QueryValid; import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery; import org.breedinginsight.brapi.v2.services.BrAPITrialService; +import org.breedinginsight.brapps.importer.model.base.ObservationVariable; +import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.DownloadFile; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.response.ResponseUtils; import org.breedinginsight.utilities.response.mappers.ExperimentQueryMapper; @@ -35,14 +42,17 @@ @Controller @Secured(SecurityRule.IS_AUTHENTICATED) public class ExperimentController { - + private final ProgramDAO programDAO; private final BrAPITrialService experimentService; private final ExperimentQueryMapper experimentQueryMapper; + private final ProgramService programService; @Inject - public ExperimentController(BrAPITrialService experimentService, ExperimentQueryMapper experimentQueryMapper) { + public ExperimentController(ProgramDAO programDAO, BrAPITrialService experimentService, ExperimentQueryMapper experimentQueryMapper, ProgramService programService) { + this.programDAO = programDAO; this.experimentService = experimentService; this.experimentQueryMapper = experimentQueryMapper; + this.programService = programService; } @Get("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/trials{?queryParams*}") @@ -91,10 +101,14 @@ public HttpResponse> getExperimentById( @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) public HttpResponse datasetExport( @PathVariable("programId") UUID programId, @PathVariable("experimentId") UUID experimentId, - @QueryValue Object queryParams) { + @QueryValue ExperimentExportQuery queryParams) { String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; try { - DownloadFile datasetFile = experimentService.exportObservations(programId, experimentId, queryParams); + Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program does not exist")); + BrAPITrial experiment = experimentService.getExperiment(program, experimentId); + //List obsVars = experimentService.getDatasetObsVars(experiment, program); + //List dataset = experimentService.getObservationDataset(program, experimentId); + DownloadFile datasetFile = experimentService.exportObservations(program, experimentId, queryParams); HttpResponse datasetExport = HttpResponse.ok(datasetFile.getStreamedFile()).header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + datasetFile.getFileName() + ".xlsx"); return datasetExport; } catch (Exception e) { diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java new file mode 100644 index 000000000..8f81415a8 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java @@ -0,0 +1,21 @@ +package org.breedinginsight.brapi.v2.model.request.query; + +import io.micronaut.core.annotation.Introspected; +import lombok.Getter; +import org.breedinginsight.api.model.v1.request.query.FilterRequest; +import org.breedinginsight.api.model.v1.request.query.SearchRequest; +import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery; +import org.jooq.tools.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +@Getter +@Introspected +public class ExperimentExportQuery { + private String fileExtension; + private String dataset; + private String environments; + private String includeTimestamps; + +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 7a33f1aaf..ebf30133d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -4,21 +4,32 @@ import io.micronaut.http.server.types.files.StreamedFile; import org.brapi.client.v2.model.exceptions.ApiException; import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.BrAPIListTypes; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.Utilities; +import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationVariable; +import org.brapi.v2.model.pheno.BrAPITraitDataType; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; +import org.breedinginsight.brapps.importer.daos.BrAPIListDAO; import org.breedinginsight.brapps.importer.daos.BrAPIObservationDAO; +import org.breedinginsight.brapps.importer.daos.BrAPIObservationVariableDAO; import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; import org.breedinginsight.brapps.importer.model.base.AdditionalInfo; +import org.breedinginsight.brapps.importer.model.base.ObservationVariable; import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.model.Column; import org.breedinginsight.model.DownloadFile; +import org.breedinginsight.model.Program; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; @@ -40,14 +51,22 @@ public class BrAPITrialService { private final BrAPIObservationUnitDAO ouDAO; private final BrAPIObservationDAO observationDAO; private final ProgramService programService; + private final BrAPIListDAO listDAO; + private final BrAPIObservationVariableDAO obsVarDAO; @Inject - public BrAPITrialService(ProgramService programService, BrAPITrialDAO trialDAO, BrAPIObservationUnitDAO ouDAO, BrAPIObservationDAO observationDAO) { + public BrAPITrialService(ProgramService programService, + BrAPITrialDAO trialDAO, + BrAPIObservationUnitDAO ouDAO, + BrAPIObservationDAO observationDAO, + BrAPIListDAO listDAO, + BrAPIObservationVariableDAO obsVarDAO) { this.programService = programService; this.trialDAO = trialDAO; this.ouDAO = ouDAO; this.observationDAO = observationDAO; - + this.listDAO = listDAO; + this.obsVarDAO = obsVarDAO; } public List getExperiments(UUID programId) throws ApiException, DoesNotExistException { @@ -74,21 +93,97 @@ public BrAPITrial getTrialDataByUUID(UUID programId, UUID trialId, boolean stats private long countGermplasm(UUID programId, String trialDbId) throws ApiException, DoesNotExistException{ List obUnits = ouDAO.getObservationUnitsForTrialDbId(programId, trialDbId); return obUnits.stream().map(BrAPIObservationUnit::getGermplasmDbId).distinct().count(); + public List getDatasetObsVars(String datasetId, Program program) throws ApiException, DoesNotExistException { + List lists = listDAO.getListByTypeAndExternalRef( + BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", referenceSource, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(datasetId)); + if (lists == null || lists.isEmpty()) { + throw new DoesNotExistException("Dataset observation variables list not returned from BrAPI service"); + } + String listDbId = lists.get(0).getListDbId(); + BrAPIListsSingleResponse list = listDAO.getListById(listDbId, program.getId()); + List obsVarNames = list.getResult().getData(); + List obsVars = obsVarDAO.getVariableByName(obsVarNames, program.getId()); + return obsVars; + } + + public BrAPITrial getExperiment(Program program, UUID experimentId) throws ApiException { + List experiments = trialDAO.getTrialsByExperimentIds(List.of(experimentId), program); + return experiments.get(0); + } + + private List> addBrAPIObsToRecords(List> maps, List dataset) { + for (BrAPIObservation obs: dataset) { + HashMap row = new HashMap<>(); + row.put("Germplasm Name", obs.getGermplasmName()); + row.put("Exp Unit ID", obs.getObservationUnitName()); + row.put("Env", obs.getAdditionalInfo().getAsJsonObject().get("studyName").getAsString()); + + maps.add(row); + } + return maps; + } + + private List addObsVarColumns(List columns, List obsVars) { + for (BrAPIObservationVariable var: obsVars) { + Column obsVarColumn = new Column(); + obsVarColumn.setDataType(Column.ColumnDataType.STRING); + if (var.getScale().getDataType().equals(BrAPITraitDataType.ORDINAL)) { + obsVarColumn.setDataType(Column.ColumnDataType.INTEGER); + } + if (var.getScale().getDataType().equals(BrAPITraitDataType.NUMERICAL) || + var.getScale().getDataType().equals(BrAPITraitDataType.DURATION)) { + obsVarColumn.setDataType(Column.ColumnDataType.DOUBLE); + } + obsVarColumn.setValue(var.getObservationVariableName()); + columns.add(obsVarColumn); + } + return columns; + } private String makeFileName() { // _Observation Dataset [-]__ return "Exp_Observation Dataset [KEY-EXP]_ENV"; } - public DownloadFile exportObservations(UUID programId, UUID experimentId, Object queryParams ) throws IOException { + private List filterDatasetByEnvironment(List dataset, List envNames) { + return dataset.stream().filter(obs -> envNames.contains( + obs.getAdditionalInfo().getAsJsonObject() + .get("studyName").getAsString())).collect(Collectors.toList()); + } + public DownloadFile exportObservations( + Program program, + UUID experimentId, + ExperimentExportQuery params + ) throws IOException, DoesNotExistException, ApiException { + List dataset = getObservationDataset(program, experimentId); + if (params.getEnvironments() != null) { + List envNames = new ArrayList<>(Arrays.asList(params.getEnvironments().split(","))); + dataset = filterDatasetByEnvironment(dataset, envNames); + } + BrAPITrial experiment = getExperiment(program, experimentId); + String obsDatasetId = experiment + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID.toString()).getAsString(); List columns = ExperimentFileColumns.getOrderedColumns(); + if (obsDatasetId != null) { + List obsVars = getDatasetObsVars(obsDatasetId, program); + columns = addObsVarColumns(columns, obsVars); + } + + + FileType fileExtension = FileType.XLSX; String fileName = makeFileName(); StreamedFile downloadFile; - List> processedData = new ArrayList<>(); + List> experimentObservationRecords = new ArrayList<>(); + + experimentObservationRecords = addBrAPIObsToRecords(experimentObservationRecords, dataset); if (fileExtension.equals(FileType.CSV)){ - downloadFile = CSVWriter.writeToDownload(columns, processedData, fileExtension); + downloadFile = CSVWriter.writeToDownload(columns, experimentObservationRecords, fileExtension); } else { - downloadFile = ExcelWriter.writeToDownload("Dataset Export", columns, processedData, fileExtension); + downloadFile = ExcelWriter.writeToDownload("Dataset Export", columns, experimentObservationRecords, fileExtension); } return new DownloadFile(fileName, downloadFile); @@ -115,10 +210,35 @@ private Map makeExpObsMapByHeader(ExperimentObservation obs) { return row; } - public List getDataset(UUID programId, UUID experimentId, UUID datasetId) throws ApiException, DoesNotExistException { + public List getObservationDataset(Program program, UUID experimentId) throws DoesNotExistException { + List dataset = new ArrayList<>(); + try { + List trials = trialDAO.getTrialsByExperimentIds(List.of(experimentId), program); + if (trials.isEmpty()) { + throw new DoesNotExistException("Experiment not found"); + } + BrAPITrial datasetTrial = trials.stream().filter(trial -> { + String id = trial + .getAdditionalInfo() + .getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + return id != null; + }).findAny().orElseThrow(() -> new DoesNotExistException("Experiment dataset not found")); + dataset = observationDAO.getObservationsByTrialDbId(List.of(datasetTrial.getTrialDbId()), program); + } catch (ApiException e) { + log.error("Error fetching BrAPI observations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } catch (DoesNotExistException e) { + log.error("Trial does not exist", e); + throw new DoesNotExistException(e.getMessage()); + } + return dataset; + } + public List getDataset(Program program, UUID experimentId, UUID datasetId) throws ApiException, DoesNotExistException { List dataset = new ArrayList<>(); try { - List trials = trialDAO.getTrialsByExperimentIds(List.of(experimentId), programId); + List trials = trialDAO.getTrialsByExperimentIds(List.of(experimentId), program); if (trials.isEmpty()) { throw new DoesNotExistException("Experiment not found"); } @@ -128,7 +248,7 @@ public List getDataset(UUID programId, UUID experimentId, UUID .getAsJsonObject(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); return id != null; }).findAny().orElseThrow(() -> new DoesNotExistException("Experiment dataset not found")); - dataset = observationDAO.getObservationsByTrialDbId(List.of(datasetTrial.getTrialDbId()), programId); + dataset = observationDAO.getObservationsByTrialDbId(List.of(datasetTrial.getTrialDbId()), program); } catch (ApiException e) { log.error("Error fetching BrAPI observations: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java index 6085341fe..7083cae0e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java @@ -69,18 +69,18 @@ public List getObservationsByStudyName(List studyNames ); } - public List getObservationsByTrialDbId(List trialDbIds, UUID programId) throws ApiException { + public List getObservationsByTrialDbId(List trialDbIds, Program program) throws ApiException { if(trialDbIds.isEmpty()) { return Collections.emptyList(); } BrAPIObservationSearchRequest observationSearchRequest = new BrAPIObservationSearchRequest(); - observationSearchRequest.setProgramDbIds(List.of(programId.toString())); + observationSearchRequest.setProgramDbIds(List.of(program.getBrapiProgram().getProgramDbId())); observationSearchRequest.setTrialDbIds(new ArrayList<>(trialDbIds)); - ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); + ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationsApi.class); return brAPIDAOUtil.search( api::searchObservationsPost, - (brAPIWSMIMEDataTypes, searchResultsDbId, page, pageSize) -> searchObservationsSearchResultsDbIdGet(programId, searchResultsDbId, page, pageSize), + (brAPIWSMIMEDataTypes, searchResultsDbId, page, pageSize) -> searchObservationsSearchResultsDbIdGet(program.getId(), searchResultsDbId, page, pageSize), observationSearchRequest ); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index 7a2d2485c..0a9180936 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -164,16 +164,17 @@ public List getTrialsByDbIds(Collection trialDbIds, Program ); } - public List getTrialsByExperimentIds(Collection trialExperimentIds, UUID programId) throws ApiException { - if(trialExperimentIds.isEmpty()) { + public List getTrialsByExperimentIds(Collection experimentIds, Program program) throws ApiException { + if(experimentIds.isEmpty()) { return Collections.emptyList(); } BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); - trialSearch.programDbIds(List.of(programId.toString())); - trialSearch.externalReferenceSources(List.of(ExternalReferenceSource.TRIALS.getName())); - trialSearch.externalReferenceIDs(trialExperimentIds.stream().map(id -> id.toString()).collect(Collectors.toList())); - TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); + trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + //trialSearch.trialDbIds(experimentIds.stream().map(id -> id.toString()).collect(Collectors.toList())); + trialSearch.externalReferenceSources(List.of(referenceSource + "/" + ExternalReferenceSource.TRIALS.getName())); + trialSearch.externalReferenceIDs(experimentIds.stream().map(id -> id.toString()).collect(Collectors.toList())); + TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); return brAPIDAOUtil.search( api::searchTrialsPost, api::searchTrialsSearchResultsDbIdGet, diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index 67b8c5d9c..d2dcc88f3 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -23,6 +23,7 @@ import org.apache.poi.xssf.usermodel.XSSFWorkbookFactory; import org.apache.xmlbeans.impl.xb.ltgfmt.TestsDocument; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIProgram; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservation; @@ -155,7 +156,7 @@ void setup() throws Exception { .get(0).getAsJsonObject().get("id").getAsString(); // Add traits to program - traits = createTraits(1); + traits = createTraits(2); AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); try { ontologyService.createTraits(program.getId(), traits, user, false); @@ -195,7 +196,6 @@ void setup() throws Exception { .get(0).getAsJsonObject() .get("trial").getAsJsonObject() .get("id").getAsString(); - } private File writeDataToFile(List> data, List traits) throws IOException { From 95f00b6c983e46580fd91971b1a6b2fd1d25ebef Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 23 May 2023 15:36:01 -0400 Subject: [PATCH 06/27] cast column values --- .../brapi/v2/ExperimentController.java | 23 ++- .../request/query/ExperimentExportQuery.java | 3 + .../brapi/v2/services/BrAPITrialService.java | 192 ++++++++++++++---- .../experiment/ExperimentFileColumns.java | 8 +- .../ExperimentControllerIntegrationTest.java | 2 +- 5 files changed, 184 insertions(+), 44 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index 508a3d9a2..fe5095c7c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -24,6 +24,7 @@ import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery; import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.brapps.importer.model.base.ObservationVariable; +import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.Program; @@ -99,18 +100,30 @@ public HttpResponse> getExperimentById( @Get("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/export{?queryParams*}") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + @Produces({"text/csv", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}) public HttpResponse datasetExport( @PathVariable("programId") UUID programId, @PathVariable("experimentId") UUID experimentId, @QueryValue ExperimentExportQuery queryParams) { String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; try { Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program does not exist")); - BrAPITrial experiment = experimentService.getExperiment(program, experimentId); - //List obsVars = experimentService.getDatasetObsVars(experiment, program); - //List dataset = experimentService.getObservationDataset(program, experimentId); + //BrAPITrial experiment = experimentService.getExperiment(program, experimentId); DownloadFile datasetFile = experimentService.exportObservations(program, experimentId, queryParams); - HttpResponse datasetExport = HttpResponse.ok(datasetFile.getStreamedFile()).header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + datasetFile.getFileName() + ".xlsx"); - return datasetExport; + FileType type = FileType.XLSX; + if (queryParams.getFileExtension().equals(FileType.CSV.getName())) { + type = FileType.CSV; + } + if (queryParams.getFileExtension().equals(FileType.XLS.getName())) { + type = FileType.XLS; + } + + HttpResponse response = HttpResponse + .status(HttpStatus.OK) + .contentType(type.getMimeType()) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + datasetFile.getFileName()) + .body(datasetFile.getStreamedFile()); + //HttpResponse datasetExport = HttpResponse.ok(datasetFile.getStreamedFile()).header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + datasetFile.getFileName()); + return response; } catch (Exception e) { log.info(e.getMessage(), e); e.printStackTrace(); diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java index 8f81415a8..02134d479 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java @@ -5,8 +5,10 @@ import org.breedinginsight.api.model.v1.request.query.FilterRequest; import org.breedinginsight.api.model.v1.request.query.SearchRequest; import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery; +import org.breedinginsight.brapps.importer.model.exports.FileType; import org.jooq.tools.StringUtils; +import javax.validation.constraints.NotNull; import java.util.ArrayList; import java.util.List; @@ -16,6 +18,7 @@ public class ExperimentExportQuery { private String fileExtension; private String dataset; private String environments; + @NotNull private String includeTimestamps; } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index ebf30133d..639305b21 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -4,8 +4,10 @@ import io.micronaut.http.server.types.files.StreamedFile; import org.brapi.client.v2.model.exceptions.ApiException; import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIListTypes; +import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; @@ -13,20 +15,18 @@ import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.Utilities; import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; -import org.brapi.v2.model.pheno.BrAPIObservation; -import org.brapi.v2.model.pheno.BrAPIObservationVariable; -import org.brapi.v2.model.pheno.BrAPITraitDataType; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.*; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; -import org.breedinginsight.brapps.importer.daos.BrAPIListDAO; -import org.breedinginsight.brapps.importer.daos.BrAPIObservationDAO; -import org.breedinginsight.brapps.importer.daos.BrAPIObservationVariableDAO; -import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.base.AdditionalInfo; import org.breedinginsight.brapps.importer.model.base.ObservationVariable; import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.BrAPIConstants; import org.breedinginsight.model.Column; import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.Program; @@ -40,6 +40,8 @@ import javax.inject.Inject; import javax.inject.Singleton; import java.io.IOException; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.*; import java.util.stream.Collectors; @@ -53,6 +55,9 @@ public class BrAPITrialService { private final ProgramService programService; private final BrAPIListDAO listDAO; private final BrAPIObservationVariableDAO obsVarDAO; + private final BrAPIStudyDAO studyDAO; + private final BrAPIObservationUnitDAO ouDAO; + private final BrAPIGermplasmDAO germplasmDAO; @Inject public BrAPITrialService(ProgramService programService, @@ -60,13 +65,19 @@ public BrAPITrialService(ProgramService programService, BrAPIObservationUnitDAO ouDAO, BrAPIObservationDAO observationDAO, BrAPIListDAO listDAO, - BrAPIObservationVariableDAO obsVarDAO) { + BrAPIObservationVariableDAO obsVarDAO, + BrAPIStudyDAO studyDAO, + BrAPIObservationUnitDAO ouDAO, + BrAPIGermplasmDAO germplasmDAO) { this.programService = programService; this.trialDAO = trialDAO; this.ouDAO = ouDAO; this.observationDAO = observationDAO; this.listDAO = listDAO; this.obsVarDAO = obsVarDAO; + this.studyDAO = studyDAO; + this.ouDAO = ouDAO; + this.germplasmDAO = germplasmDAO; } public List getExperiments(UUID programId) throws ApiException, DoesNotExistException { @@ -114,19 +125,111 @@ public BrAPITrial getExperiment(Program program, UUID experimentId) throws ApiEx return experiments.get(0); } - private List> addBrAPIObsToRecords(List> maps, List dataset) { + private void addObsVarDataToRow(Map row, BrAPIObservation obs, boolean includeTimestamp, List obsVars) { + // get observation variable for BrAPI observation + BrAPIObservationVariable var = obsVars.stream() + .filter(obsVar -> obs.getObservationVariableName().equals(obsVar.getObservationVariableName())) + .collect(Collectors.toList()).get(0); + + if (var.getScale().getDataType().equals(BrAPITraitDataType.ORDINAL)) { + row.put(obs.getObservationVariableName(), Integer.parseInt(obs.getValue())); + } else if (var.getScale().getDataType().equals(BrAPITraitDataType.NUMERICAL) || + var.getScale().getDataType().equals(BrAPITraitDataType.DURATION)) { + row.put(obs.getObservationVariableName(), Double.parseDouble(obs.getValue())); + } else { + row.put(obs.getObservationVariableName(), obs.getValue()); + } + + if (includeTimestamp) { + row.put(String.format("TS:%s",obs.getObservationVariableName()), obs.getObservationTimeStamp()); + } + } + private List> addBrAPIObsToRecords( + List> maps, + List dataset, + BrAPITrial experiment, + Program program, + boolean includeTimestamp, + List obsVars) throws ApiException, DoesNotExistException { for (BrAPIObservation obs: dataset) { - HashMap row = new HashMap<>(); - row.put("Germplasm Name", obs.getGermplasmName()); - row.put("Exp Unit ID", obs.getObservationUnitName()); - row.put("Env", obs.getAdditionalInfo().getAsJsonObject().get("studyName").getAsString()); - maps.add(row); + // get ouId + List ous = ouDAO.getObservationUnitByName(List.of(obs.getObservationUnitName()), program); + if (ous.isEmpty()) { + throw new DoesNotExistException("Observation unit not returned from BrAPI service"); + } + BrAPIObservationUnit ou = ous.stream() + .filter(unit -> obs.getObservationUnitDbId().equals(unit.getObservationUnitDbId())) + .findAny() + .orElseThrow(() -> new RuntimeException()); + BrAPIExternalReference ouXref = Utilities.getExternalReference( + ou.getExternalReferences(), + String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName())) + .orElseThrow(() -> new RuntimeException("observation unit id not found")); + String ouId = ouXref.getReferenceID(); + + // if there is a row with that ouId then just add the obs var data and timestamp to the row + Optional> existingRow = maps.stream() + .filter(row -> ouId.equals(row.get(ExperimentObservation.Columns.OBS_UNIT_ID))) + .findAny(); + if (existingRow.isPresent()) { + addObsVarDataToRow(existingRow.get(), obs, includeTimestamp, obsVars); + } else { + + // otherwise, make a new row + HashMap row = new HashMap<>(); + BrAPIGermplasm germplasm = germplasmDAO.getGermplasmByDBID(obs.getGermplasmDbId(), program.getId()) + .orElseThrow(() -> new DoesNotExistException("Germplasm not returned from BrAPI service")); + BrAPIStudy study = studyDAO.getStudyByDbId(obs.getStudyDbId(), program) + .orElseThrow(() -> new DoesNotExistException("Study not returned from BrAPI Service")); + row.put(ExperimentObservation.Columns.GERMPLASM_NAME, obs.getGermplasmName()); + row.put(ExperimentObservation.Columns.GERMPLASM_GID, germplasm.getAccessionNumber()); + row.put(ExperimentObservation.Columns.TEST_CHECK, ou.getObservationUnitPosition().getEntryType().toString()); + row.put(ExperimentObservation.Columns.EXP_TITLE, experiment.getTrialName()); + row.put(ExperimentObservation.Columns.EXP_DESCRIPTION, experiment.getTrialDescription()); + row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); + row.put(ExperimentObservation.Columns.ENV, obs.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.STUDY_NAME).getAsString()); + row.put(ExperimentObservation.Columns.ENV_LOCATION, study.getLocationName()); + row.put(ExperimentObservation.Columns.ENV_YEAR, obs.getSeason().getYear()); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, obs.getObservationUnitName()); + + // get replicate number + Optional repLevel = ou.getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(level -> BrAPIConstants.REPLICATE.getValue().equals(level.getLevelName())) + .findFirst(); + if (repLevel.isPresent()) { + row.put(ExperimentObservation.Columns.REP_NUM, Integer.parseInt(repLevel.get().getLevelCode())); + } + + //get block number + Optional blockLevel = ou.getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(level -> BrAPIConstants.BLOCK.getValue().equals(level.getLevelName())) + .findFirst(); + if (blockLevel.isPresent()) { + row.put(ExperimentObservation.Columns.BLOCK_NUM, Integer.parseInt(blockLevel.get().getLevelCode())); + } + + if (ou.getObservationUnitPosition() != null) { + row.put(ExperimentObservation.Columns.ROW, Double.parseDouble(ou.getObservationUnitPosition().getPositionCoordinateX())); + row.put(ExperimentObservation.Columns.COLUMN, Double.parseDouble(ou.getObservationUnitPosition().getPositionCoordinateY())); + } + + if (ou.getTreatments() != null && !ou.getTreatments().isEmpty()) { + row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, ou.getTreatments().get(0).getFactor().toString()); + } + + row.put(ExperimentObservation.Columns.OBS_UNIT_ID, ouId); + addObsVarDataToRow(row, obs, includeTimestamp, obsVars); + maps.add(row); + } } return maps; } - private List addObsVarColumns(List columns, List obsVars) { + private List addObsVarColumns(List columns, List obsVars, boolean includeTimestamps) { for (BrAPIObservationVariable var: obsVars) { Column obsVarColumn = new Column(); obsVarColumn.setDataType(Column.ColumnDataType.STRING); @@ -139,53 +242,74 @@ private List addObsVarColumns(List columns, List_Observation Dataset [-]__ - return "Exp_Observation Dataset [KEY-EXP]_ENV"; + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + return String.format("%s_Observation Dataset [%s-%s]_%s_%s", + experiment.getTrialName(), + program.getKey(), + experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString(), + envName, + timestamp); } private List filterDatasetByEnvironment(List dataset, List envNames) { return dataset.stream().filter(obs -> envNames.contains( obs.getAdditionalInfo().getAsJsonObject() - .get("studyName").getAsString())).collect(Collectors.toList()); + .get(BrAPIAdditionalInfoFields.STUDY_NAME).getAsString())).collect(Collectors.toList()); } public DownloadFile exportObservations( Program program, UUID experimentId, - ExperimentExportQuery params - ) throws IOException, DoesNotExistException, ApiException { + ExperimentExportQuery params) throws IOException, DoesNotExistException, ApiException { + // process params + boolean includeTimestamps = params.getIncludeTimestamps().equals("true"); + FileType fileType = FileType.XLSX; + if (params.getFileExtension().equals(FileType.CSV.getName())) { + fileType = FileType.CSV; + } + if (params.getFileExtension().equals(FileType.XLS.getName())) { + fileType = FileType.XLS; + } + + // get BrAPI observations for requested environments List dataset = getObservationDataset(program, experimentId); if (params.getEnvironments() != null) { List envNames = new ArrayList<>(Arrays.asList(params.getEnvironments().split(","))); dataset = filterDatasetByEnvironment(dataset, envNames); } + + // add columns in the export for any observation variables BrAPITrial experiment = getExperiment(program, experimentId); - String obsDatasetId = experiment - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID.toString()).getAsString(); + List obsVars = new ArrayList<>(); List columns = ExperimentFileColumns.getOrderedColumns(); - if (obsDatasetId != null) { - List obsVars = getDatasetObsVars(obsDatasetId, program); - columns = addObsVarColumns(columns, obsVars); + if (experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null) { + String obsDatasetId = experiment + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); + obsVars = getDatasetObsVars(obsDatasetId, program); + columns = addObsVarColumns(columns, obsVars, includeTimestamps); } - - - FileType fileExtension = FileType.XLSX; - String fileName = makeFileName(); StreamedFile downloadFile; List> experimentObservationRecords = new ArrayList<>(); - experimentObservationRecords = addBrAPIObsToRecords(experimentObservationRecords, dataset); - if (fileExtension.equals(FileType.CSV)){ - downloadFile = CSVWriter.writeToDownload(columns, experimentObservationRecords, fileExtension); + experimentObservationRecords = addBrAPIObsToRecords(experimentObservationRecords, dataset, experiment, program, includeTimestamps, obsVars); + + if (params.getFileExtension().equals(FileType.CSV)){ + downloadFile = CSVWriter.writeToDownload(columns, experimentObservationRecords, fileType); } else { - downloadFile = ExcelWriter.writeToDownload("Dataset Export", columns, experimentObservationRecords, fileExtension); + downloadFile = ExcelWriter.writeToDownload("Dataset Export", columns, experimentObservationRecords, fileType); } + String fileName = makeFileName(experiment, program, params.getEnvironments()) + fileType.getExtension(); return new DownloadFile(fileName, downloadFile); } diff --git a/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java index beb533034..d12da2399 100644 --- a/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java +++ b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java @@ -27,11 +27,11 @@ public enum ExperimentFileColumns { GERMPLASM_NAME(ExperimentObservation.Columns.GERMPLASM_NAME, Column.ColumnDataType.STRING), - GERMPLASM_GID(ExperimentObservation.Columns.GERMPLASM_GID, Column.ColumnDataType.INTEGER), + GERMPLASM_GID(ExperimentObservation.Columns.GERMPLASM_GID, Column.ColumnDataType.STRING), TEST_CHECK(ExperimentObservation.Columns.TEST_CHECK, Column.ColumnDataType.STRING), EXP_TITLE(ExperimentObservation.Columns.EXP_TITLE, Column.ColumnDataType.STRING), EXP_DESCRIPTION(ExperimentObservation.Columns.EXP_DESCRIPTION, Column.ColumnDataType.STRING), - EXP_UNIT(ExperimentObservation.Columns.EXP_UNIT, Column.ColumnDataType.INTEGER), + EXP_UNIT(ExperimentObservation.Columns.EXP_UNIT, Column.ColumnDataType.STRING), EXP_TYPE(ExperimentObservation.Columns.EXP_TYPE, Column.ColumnDataType.STRING), ENV(ExperimentObservation.Columns.ENV, Column.ColumnDataType.STRING), ENV_LOCATION(ExperimentObservation.Columns.ENV_LOCATION, Column.ColumnDataType.STRING), @@ -39,8 +39,8 @@ public enum ExperimentFileColumns { EXP_UNIT_ID(ExperimentObservation.Columns.EXP_UNIT_ID, Column.ColumnDataType.STRING), REP_NUM(ExperimentObservation.Columns.REP_NUM, Column.ColumnDataType.INTEGER), BLOCK_NUM(ExperimentObservation.Columns.BLOCK_NUM, Column.ColumnDataType.INTEGER), - ROW(ExperimentObservation.Columns.ROW, Column.ColumnDataType.INTEGER), - COLUMN(ExperimentObservation.Columns.COLUMN, Column.ColumnDataType.INTEGER), + ROW(ExperimentObservation.Columns.ROW, Column.ColumnDataType.DOUBLE), + COLUMN(ExperimentObservation.Columns.COLUMN, Column.ColumnDataType.DOUBLE), TREATMENT_FACTORS(ExperimentObservation.Columns.TREATMENT_FACTORS, Column.ColumnDataType.STRING), OBS_UNIT_ID(ExperimentObservation.Columns.OBS_UNIT_ID, Column.ColumnDataType.STRING); diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index d2dcc88f3..77c3cf3b1 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -448,7 +448,7 @@ void downloadDatasets(boolean hasEmptyObs, String requestedEnv, String extension envParam = "env=" + requestedEnv; } Flowable> call = client.exchange( - GET(String.format("/programs/%s/experiments/%s/export?%s&fileExtension=%s",program.getId().toString(), experimentId, envParam, extension)) + GET(String.format("/programs/%s/experiments/%s/export?includeTimestamps=true&%s&fileExtension=%s",program.getId().toString(), experimentId, envParam, extension)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class ); HttpResponse response = call.blockingFirst(); From db1f8e5bf317fa50259c567126805ba559b8578f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 25 May 2023 14:47:31 -0400 Subject: [PATCH 07/27] fix test parsing of download --- .../brapi/v2/ExperimentController.java | 20 +-- .../brapi/v2/services/BrAPITrialService.java | 59 +++++--- .../ExperimentControllerIntegrationTest.java | 127 +++++++----------- 3 files changed, 96 insertions(+), 110 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index fe5095c7c..47ae1e114 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -32,6 +32,9 @@ import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.response.ResponseUtils; import org.breedinginsight.utilities.response.mappers.ExperimentQueryMapper; +import org.breedinginsight.utilities.FileUtil; +import org.breedinginsight.utilities.Utilities; +import tech.tablesaw.api.Table; import javax.inject.Inject; import javax.validation.Valid; @@ -107,22 +110,11 @@ public HttpResponse datasetExport( String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; try { Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program does not exist")); - //BrAPITrial experiment = experimentService.getExperiment(program, experimentId); DownloadFile datasetFile = experimentService.exportObservations(program, experimentId, queryParams); - FileType type = FileType.XLSX; - if (queryParams.getFileExtension().equals(FileType.CSV.getName())) { - type = FileType.CSV; - } - if (queryParams.getFileExtension().equals(FileType.XLS.getName())) { - type = FileType.XLS; - } - + Table table = FileUtil.parseTableFromCsv(datasetFile.getStreamedFile().getInputStream()); HttpResponse response = HttpResponse - .status(HttpStatus.OK) - .contentType(type.getMimeType()) - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + datasetFile.getFileName()) - .body(datasetFile.getStreamedFile()); - //HttpResponse datasetExport = HttpResponse.ok(datasetFile.getStreamedFile()).header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + datasetFile.getFileName()); + .ok(datasetFile.getStreamedFile()) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + datasetFile.getFileName()); return response; } catch (Exception e) { log.info(e.getMessage(), e); diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 639305b21..8df10da65 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -32,13 +32,17 @@ import org.breedinginsight.model.Program; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.parsers.ParsingException; import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; import org.breedinginsight.services.writers.CSVWriter; import org.breedinginsight.services.writers.ExcelWriter; +import org.breedinginsight.utilities.FileUtil; import org.breedinginsight.utilities.Utilities; +import tech.tablesaw.api.Table; import javax.inject.Inject; import javax.inject.Singleton; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; @@ -125,23 +129,29 @@ public BrAPITrial getExperiment(Program program, UUID experimentId) throws ApiEx return experiments.get(0); } - private void addObsVarDataToRow(Map row, BrAPIObservation obs, boolean includeTimestamp, List obsVars) { + private void addObsVarDataToRow( + Map row, + BrAPIObservation obs, + boolean includeTimestamp, + List obsVars, + Program program) { // get observation variable for BrAPI observation BrAPIObservationVariable var = obsVars.stream() .filter(obsVar -> obs.getObservationVariableName().equals(obsVar.getObservationVariableName())) .collect(Collectors.toList()).get(0); + String varName = Utilities.removeProgramKey(obs.getObservationVariableName(), program.getKey()); if (var.getScale().getDataType().equals(BrAPITraitDataType.ORDINAL)) { - row.put(obs.getObservationVariableName(), Integer.parseInt(obs.getValue())); + row.put(varName, Integer.parseInt(obs.getValue())); } else if (var.getScale().getDataType().equals(BrAPITraitDataType.NUMERICAL) || var.getScale().getDataType().equals(BrAPITraitDataType.DURATION)) { - row.put(obs.getObservationVariableName(), Double.parseDouble(obs.getValue())); + row.put(varName, Double.parseDouble(obs.getValue())); } else { - row.put(obs.getObservationVariableName(), obs.getValue()); + row.put(varName, obs.getValue()); } if (includeTimestamp) { - row.put(String.format("TS:%s",obs.getObservationVariableName()), obs.getObservationTimeStamp()); + row.put(String.format("TS:%s",varName), obs.getObservationTimeStamp()); } } private List> addBrAPIObsToRecords( @@ -173,7 +183,7 @@ private List> addBrAPIObsToRecords( .filter(row -> ouId.equals(row.get(ExperimentObservation.Columns.OBS_UNIT_ID))) .findAny(); if (existingRow.isPresent()) { - addObsVarDataToRow(existingRow.get(), obs, includeTimestamp, obsVars); + addObsVarDataToRow(existingRow.get(), obs, includeTimestamp, obsVars, program); } else { // otherwise, make a new row @@ -219,17 +229,23 @@ private List> addBrAPIObsToRecords( if (ou.getTreatments() != null && !ou.getTreatments().isEmpty()) { row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, ou.getTreatments().get(0).getFactor().toString()); + } else { + row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, null); } row.put(ExperimentObservation.Columns.OBS_UNIT_ID, ouId); - addObsVarDataToRow(row, obs, includeTimestamp, obsVars); + addObsVarDataToRow(row, obs, includeTimestamp, obsVars, program); maps.add(row); } } return maps; } - private List addObsVarColumns(List columns, List obsVars, boolean includeTimestamps) { + private List addObsVarColumns( + List columns, + List obsVars, + boolean includeTimestamps, + Program program) { for (BrAPIObservationVariable var: obsVars) { Column obsVarColumn = new Column(); obsVarColumn.setDataType(Column.ColumnDataType.STRING); @@ -240,10 +256,11 @@ private List addObsVarColumns(List columns, List filterDatasetByEnvironment(List public DownloadFile exportObservations( Program program, UUID experimentId, - ExperimentExportQuery params) throws IOException, DoesNotExistException, ApiException { + ExperimentExportQuery params) throws IOException, DoesNotExistException, ApiException, ParsingException { // process params boolean includeTimestamps = params.getIncludeTimestamps().equals("true"); FileType fileType = FileType.XLSX; - if (params.getFileExtension().equals(FileType.CSV.getName())) { + if (params.getFileExtension().toLowerCase().equals(FileType.CSV.getName())) { fileType = FileType.CSV; } - if (params.getFileExtension().equals(FileType.XLS.getName())) { + if (params.getFileExtension().toLowerCase().equals(FileType.XLS.getName())) { fileType = FileType.XLS; } @@ -294,7 +311,7 @@ public DownloadFile exportObservations( .getAdditionalInfo().getAsJsonObject() .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); obsVars = getDatasetObsVars(obsDatasetId, program); - columns = addObsVarColumns(columns, obsVars, includeTimestamps); + columns = addObsVarColumns(columns, obsVars, includeTimestamps, program); } StreamedFile downloadFile; @@ -303,13 +320,15 @@ public DownloadFile exportObservations( experimentObservationRecords = addBrAPIObsToRecords(experimentObservationRecords, dataset, experiment, program, includeTimestamps, obsVars); - if (params.getFileExtension().equals(FileType.CSV)){ + if (fileType.equals(FileType.CSV)){ downloadFile = CSVWriter.writeToDownload(columns, experimentObservationRecords, fileType); } else { downloadFile = ExcelWriter.writeToDownload("Dataset Export", columns, experimentObservationRecords, fileType); } - String fileName = makeFileName(experiment, program, params.getEnvironments()) + fileType.getExtension(); + // Table table = FileUtil.parseTableFromCsv(downloadFile.getInputStream()); + String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); + String fileName = makeFileName(experiment, program, envFilenameFragment) + fileType.getExtension(); return new DownloadFile(fileName, downloadFile); } @@ -342,13 +361,11 @@ public List getObservationDataset(Program program, UUID experi throw new DoesNotExistException("Experiment not found"); } BrAPITrial datasetTrial = trials.stream().filter(trial -> { - String id = trial + return trial .getAdditionalInfo() .getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) - .getAsString(); - return id != null; - }).findAny().orElseThrow(() -> new DoesNotExistException("Experiment dataset not found")); + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null; + }).findFirst().orElseThrow(() -> new DoesNotExistException("Experiment dataset not found")); dataset = observationDAO.getObservationsByTrialDbId(List.of(datasetTrial.getTrialDbId()), program); } catch (ApiException e) { log.error("Error fetching BrAPI observations: " + Utilities.generateApiExceptionLogMessage(e), e); diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index 77c3cf3b1..1b9bf478e 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -27,6 +27,7 @@ import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPITrait; import org.breedinginsight.BrAPITest; import org.breedinginsight.TestUtils; import org.breedinginsight.api.auth.AuthenticatedUser; @@ -36,6 +37,7 @@ import org.breedinginsight.api.v1.controller.TestTokenValidator; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.ImportTestUtils; +import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; @@ -181,8 +183,10 @@ void setup() throws Exception { // Add test observation data for (int i = 0; i < traits.size(); i++) { - row1.put(traits.get(i).getObservationVariableName(), Integer.toString(i)); - row2.put(traits.get(i).getObservationVariableName(), Integer.toString(i)); + Double val1 = Math.random(); + Double val2 = Math.random(); + row1.put(traits.get(i).getObservationVariableName(), val1); + row2.put(traits.get(i).getObservationVariableName(), val2); }; rows.add(row1); @@ -300,40 +304,20 @@ private List createGermplasm(int numToCreate) { return germplasm; } - - private void checkXLSDownload(Map dataRow, Map emptyRow, HSSFWorkbook workbook) { - HSSFSheet experimentData = workbook.getSheet("Experiment Data"); - // Filename is correct: _Observation Dataset [-]__ - // All columns included - // All environments included - // All data included - // Observation units populated - - } - - private void checkXLSXDownload(List> importRows, XSSFWorkbook workbook) { - // Filename is correct: _Observation Dataset [-]__ - // All columns included - // All environments included - // All data included - // Observation units populated - - } - - private void checkCSVDownload(List> importRows, CSVReader reader) { - // Filename is correct: _Observation Dataset [-]__ - // All columns included - // All environments included - // All data included - // Observation units populated - - } - private void checkDownloadTable(String requestedEnv, List> importRows, Table table) { + private void checkDownloadTable( + String requestedEnv, + List> importRows, + Table table, + boolean includeTimestamps) { // Filename is correct: _Observation Dataset [-]__ List> requestedImportRows; // All columns included - assertEquals(columns.size(), table.columnCount()); + Integer expectedColNumber = columns.size(); + if (includeTimestamps) { + expectedColNumber += traits.size(); + } + assertEquals(expectedColNumber, table.columnCount()); if (requestedEnv == null) { requestedImportRows = importRows; @@ -360,6 +344,19 @@ private void checkDownloadTable(String requestedEnv, List> i } // All requested import data included in download + // update columns with traitName to use "traitName [progrmKey]" + /* + List traitNames = traits.stream().map(trait -> trait.getObservationVariableName()).collect(Collectors.toList()); + for (Map row: requestedImportRows) { + row.entrySet().stream().forEach(column -> { + if (traitNames.contains(column.getKey())) { + + } + }); + } + + */ + List> matchingImportRows = requestedImportRows.stream().filter(importRow -> { for (Row downloadRow : table) { if (isMatchedRow(importRow, downloadRow)) { @@ -378,18 +375,20 @@ private void checkDownloadTable(String requestedEnv, List> i private boolean isMatchedRow(Map importRow, Row downloadRow) { return importRow.entrySet().stream().filter(e -> { String header = e.getKey(); - List importColumns = columns.stream().filter(col -> {return col.getValue() == header;}).collect(Collectors.toList()); + List importColumns = columns.stream().filter(col -> {return header.equals(col.getValue());}).collect(Collectors.toList()); if (importColumns.isEmpty() || importColumns.size() > 1) { return false; } - - if (importColumns.get(0).getDataType() == Column.ColumnDataType.STRING) { - return downloadRow.getString(e.getKey().toString()) == e.getValue(); + if (downloadRow.getColumnType(e.getKey()).equals(Column.ColumnDataType.STRING)) { + return downloadRow.getString(e.getKey()).equals(e.getValue()); } - - if (importColumns.get(0).getDataType() == Column.ColumnDataType.INTEGER) { - return downloadRow.getInt(e.getKey().toString()) == Integer.parseInt(e.getValue().toString()) ; + if (downloadRow.getColumnType(e.getKey()).equals(Column.ColumnDataType.INTEGER)) { + return downloadRow.getInt(e.getKey()) == Integer.parseInt(e.getValue().toString()); + } + if (downloadRow.getColumnType(e.getKey()).equals(Column.ColumnDataType.DOUBLE)) { + return downloadRow.getDouble(e.getKey()) == Double.parseDouble(e.getValue().toString()); } + return false; }).collect(Collectors.toList()).size() == importRow.size(); } @@ -411,44 +410,22 @@ private boolean isMatchedRow(Map importRow, Row downloadRow) { */ @ParameterizedTest - @CsvSource(value = {"true,,CSV", "false,,CSV", "true,Env1,CSV", "false,Env1,CSV", - "true,,XLS", "false,,XLS", "true,Env1,XLS", "false,Env1,XLS", - "true,,XLSX", "false,,XLSX", "true,Env1,XLSX", "false,Env1,XLSX",}) + @CsvSource(value = {"true,true,,CSV", "true,false,,CSV", "true,true,Env1,CSV", "true,false,Env1,CSV", + "false,true,,CSV", "false,false,,CSV", "false,true,Env1,CSV", "false,false,Env1,CSV", + "true,true,,XLS", "true,false,,XLS", "true,true,Env1,XLS", "true,false,Env1,XLS", + "false,true,,XLS", "false,false,,XLS", "false,true,Env1,XLS", "false,false,Env1,XLS", + "true,true,,XLSX", "true,false,,XLSX", "true,true,Env1,XLSX", "true,false,Env1,XLSX", + "false,true,,XLSX", "false,false,,XLSX", "false,true,Env1,XLSX", "false,false,Env1,XLSX",}) @SneakyThrows - void downloadDatasets(boolean hasEmptyObs, String requestedEnv, String extension) { -// // Make test experiment import -// List> rows = new ArrayList<>(); -// Map row1 = makeExpImportRow("Env1"); -// Map row2 = makeExpImportRow("Env2"); -// -// // Add test observation data -// for (int i = 0; i < traits.size(); i++) { -// row1.put(traits.get(i).getObservationVariableName(), Integer.toString(i)); -// if (!hasEmptyObs) { -// row2.put(traits.get(i).getObservationVariableName(), Integer.toString(i)); -// } -// }; -// -// rows.add(row1); -// rows.add(row2); -// -// // Import test experiment, environments, and any observations -// JsonObject importResult = importTestUtils.uploadAndFetch(writeDataToFile(rows, traits), null, true, client, program, mappingId); -// String experimentId = importResult -// .get("preview").getAsJsonObject() -// .get("rows").getAsJsonArray() -// .get(0).getAsJsonObject() -// .get("trial").getAsJsonObject() -// .get("id").getAsString(); - - + void downloadDatasets(boolean includeTimestamps, boolean hasEmptyObs, String requestedEnv, String extension) { // Download test experiment String envParam = "all=true"; if (requestedEnv != null) { - envParam = "env=" + requestedEnv; + envParam = "environments=" + requestedEnv; } Flowable> call = client.exchange( - GET(String.format("/programs/%s/experiments/%s/export?includeTimestamps=true&%s&fileExtension=%s",program.getId().toString(), experimentId, envParam, extension)) + GET(String.format("/programs/%s/experiments/%s/export?includeTimestamps=%s&%s&fileExtension=%s", + program.getId().toString(), experimentId, includeTimestamps, envParam, extension)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class ); HttpResponse response = call.blockingFirst(); @@ -458,9 +435,9 @@ void downloadDatasets(boolean hasEmptyObs, String requestedEnv, String extension // Assert file format fidelity Map mediaTypeByExtension = new HashMap<>(); - mediaTypeByExtension.put("CSV", "text/csv"); - mediaTypeByExtension.put("XLS","application/vnd.ms.excel"); - mediaTypeByExtension.put("XLSX", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + mediaTypeByExtension.put("CSV", FileType.CSV.getMimeType()); + mediaTypeByExtension.put("XLS", FileType.XLS.getMimeType()); + mediaTypeByExtension.put("XLSX", FileType.XLSX.getMimeType()); String downloadMediaType = response.getHeaders().getContentType().orElseThrow(Exception::new); assertEquals(mediaTypeByExtension.get(extension), downloadMediaType); @@ -473,6 +450,6 @@ void downloadDatasets(boolean hasEmptyObs, String requestedEnv, String extension if (extension.equals("XLS") || extension.equals("XLSX")) { download = FileUtil.parseTableFromExcel(bodyStream, 0); } - checkDownloadTable(requestedEnv, rows, download); + checkDownloadTable(requestedEnv, rows, download, includeTimestamps); } } From 03c35fb0e68afc9c7be358f5567790ed8bcb0bca Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 25 May 2023 16:11:36 -0400 Subject: [PATCH 08/27] cache studies --- .../brapi/v2/ExperimentController.java | 2 +- .../request/query/ExperimentExportQuery.java | 4 +-- .../brapi/v2/services/BrAPITrialService.java | 28 +++++++++++-------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index 47ae1e114..74b172c4a 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -103,7 +103,7 @@ public HttpResponse> getExperimentById( @Get("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/export{?queryParams*}") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - @Produces({"text/csv", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}) + @Produces(value={"text/csv", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}) public HttpResponse datasetExport( @PathVariable("programId") UUID programId, @PathVariable("experimentId") UUID experimentId, @QueryValue ExperimentExportQuery queryParams) { diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java index 02134d479..5f60f1008 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java @@ -15,10 +15,10 @@ @Getter @Introspected public class ExperimentExportQuery { - private String fileExtension; + private FileType fileExtension; private String dataset; private String environments; @NotNull - private String includeTimestamps; + private boolean includeTimestamps; } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 8df10da65..635ecc3d9 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -161,6 +161,16 @@ private List> addBrAPIObsToRecords( Program program, boolean includeTimestamp, List obsVars) throws ApiException, DoesNotExistException { + // cache studies belonging to dataset + Map studyByDbId = new HashMap<>(); + for (BrAPIObservation obs: dataset) { + if (studyByDbId.get(obs.getStudyDbId()) == null) { + BrAPIStudy study = studyDAO.getStudyByDbId(obs.getStudyDbId(), program) + .orElseThrow(() -> new DoesNotExistException("Study not returned from BrAPI Service")); + studyByDbId.put(obs.getStudyDbId(), study); + } + } + for (BrAPIObservation obs: dataset) { // get ouId @@ -190,8 +200,7 @@ private List> addBrAPIObsToRecords( HashMap row = new HashMap<>(); BrAPIGermplasm germplasm = germplasmDAO.getGermplasmByDBID(obs.getGermplasmDbId(), program.getId()) .orElseThrow(() -> new DoesNotExistException("Germplasm not returned from BrAPI service")); - BrAPIStudy study = studyDAO.getStudyByDbId(obs.getStudyDbId(), program) - .orElseThrow(() -> new DoesNotExistException("Study not returned from BrAPI Service")); + BrAPIStudy study = studyByDbId.get(obs.getStudyDbId()); row.put(ExperimentObservation.Columns.GERMPLASM_NAME, obs.getGermplasmName()); row.put(ExperimentObservation.Columns.GERMPLASM_GID, germplasm.getAccessionNumber()); row.put(ExperimentObservation.Columns.TEST_CHECK, ou.getObservationUnitPosition().getEntryType().toString()); @@ -286,14 +295,9 @@ public DownloadFile exportObservations( UUID experimentId, ExperimentExportQuery params) throws IOException, DoesNotExistException, ApiException, ParsingException { // process params - boolean includeTimestamps = params.getIncludeTimestamps().equals("true"); - FileType fileType = FileType.XLSX; - if (params.getFileExtension().toLowerCase().equals(FileType.CSV.getName())) { - fileType = FileType.CSV; - } - if (params.getFileExtension().toLowerCase().equals(FileType.XLS.getName())) { - fileType = FileType.XLS; - } + //boolean includeTimestamps = params.getIncludeTimestamps().equals("true"); + + FileType fileType = params.getFileExtension(); // get BrAPI observations for requested environments List dataset = getObservationDataset(program, experimentId); @@ -311,14 +315,14 @@ public DownloadFile exportObservations( .getAdditionalInfo().getAsJsonObject() .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); obsVars = getDatasetObsVars(obsDatasetId, program); - columns = addObsVarColumns(columns, obsVars, includeTimestamps, program); + columns = addObsVarColumns(columns, obsVars, params.isIncludeTimestamps(), program); } StreamedFile downloadFile; List> experimentObservationRecords = new ArrayList<>(); - experimentObservationRecords = addBrAPIObsToRecords(experimentObservationRecords, dataset, experiment, program, includeTimestamps, obsVars); + experimentObservationRecords = addBrAPIObsToRecords(experimentObservationRecords, dataset, experiment, program, params.isIncludeTimestamps(), obsVars); if (fileType.equals(FileType.CSV)){ downloadFile = CSVWriter.writeToDownload(columns, experimentObservationRecords, fileType); From c2718767176810f2ce589b14e8501fd45b4c76b6 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 25 May 2023 16:32:38 -0400 Subject: [PATCH 09/27] remove debugging comments --- .../org/breedinginsight/brapi/v2/ExperimentController.java | 2 +- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index 74b172c4a..550587b64 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -111,7 +111,7 @@ public HttpResponse datasetExport( try { Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program does not exist")); DownloadFile datasetFile = experimentService.exportObservations(program, experimentId, queryParams); - Table table = FileUtil.parseTableFromCsv(datasetFile.getStreamedFile().getInputStream()); + //Table table = FileUtil.parseTableFromCsv(datasetFile.getStreamedFile().getInputStream()); HttpResponse response = HttpResponse .ok(datasetFile.getStreamedFile()) .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + datasetFile.getFileName()); diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 635ecc3d9..b2064da7a 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -295,7 +295,7 @@ public DownloadFile exportObservations( UUID experimentId, ExperimentExportQuery params) throws IOException, DoesNotExistException, ApiException, ParsingException { // process params - //boolean includeTimestamps = params.getIncludeTimestamps().equals("true"); + FileType fileType = params.getFileExtension(); @@ -330,7 +330,6 @@ public DownloadFile exportObservations( downloadFile = ExcelWriter.writeToDownload("Dataset Export", columns, experimentObservationRecords, fileType); } - // Table table = FileUtil.parseTableFromCsv(downloadFile.getInputStream()); String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); String fileName = makeFileName(experiment, program, envFilenameFragment) + fileType.getExtension(); return new DownloadFile(fileName, downloadFile); From 0d22f5fbbeca20200c36d7092a1fa6422f0cadbd Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 25 May 2023 16:38:34 -0400 Subject: [PATCH 10/27] remove program key from names --- .../brapi/v2/services/BrAPITrialService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index b2064da7a..0b3dfa596 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -201,10 +201,10 @@ private List> addBrAPIObsToRecords( BrAPIGermplasm germplasm = germplasmDAO.getGermplasmByDBID(obs.getGermplasmDbId(), program.getId()) .orElseThrow(() -> new DoesNotExistException("Germplasm not returned from BrAPI service")); BrAPIStudy study = studyByDbId.get(obs.getStudyDbId()); - row.put(ExperimentObservation.Columns.GERMPLASM_NAME, obs.getGermplasmName()); + row.put(ExperimentObservation.Columns.GERMPLASM_NAME, Utilities.removeProgramKey(obs.getGermplasmName(), program.getKey())); row.put(ExperimentObservation.Columns.GERMPLASM_GID, germplasm.getAccessionNumber()); row.put(ExperimentObservation.Columns.TEST_CHECK, ou.getObservationUnitPosition().getEntryType().toString()); - row.put(ExperimentObservation.Columns.EXP_TITLE, experiment.getTrialName()); + row.put(ExperimentObservation.Columns.EXP_TITLE, Utilities.removeProgramKey(experiment.getTrialName(), program.getKey())); row.put(ExperimentObservation.Columns.EXP_DESCRIPTION, experiment.getTrialDescription()); row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); @@ -279,7 +279,7 @@ private String makeFileName(BrAPITrial experiment, Program program, String envNa DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); String timestamp = formatter.format(OffsetDateTime.now()); return String.format("%s_Observation Dataset [%s-%s]_%s_%s", - experiment.getTrialName(), + Utilities.removeProgramKey(experiment.getTrialName(), program.getKey()), program.getKey(), experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString(), envName, From b21a6ae5c4b11efd28e47d5a53217debb54edb64 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 25 May 2023 17:33:27 -0400 Subject: [PATCH 11/27] remove key from names --- .../brapi/v2/ExperimentController.java | 4 +-- .../brapi/v2/services/BrAPITrialService.java | 27 +++++++++---------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index 550587b64..b3af6676b 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -106,12 +106,12 @@ public HttpResponse> getExperimentById( @Produces(value={"text/csv", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}) public HttpResponse datasetExport( @PathVariable("programId") UUID programId, @PathVariable("experimentId") UUID experimentId, - @QueryValue ExperimentExportQuery queryParams) { + @QueryValue @Valid ExperimentExportQuery queryParams) { String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; try { Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program does not exist")); DownloadFile datasetFile = experimentService.exportObservations(program, experimentId, queryParams); - //Table table = FileUtil.parseTableFromCsv(datasetFile.getStreamedFile().getInputStream()); + Table table = FileUtil.parseTableFromCsv(datasetFile.getStreamedFile().getInputStream()); HttpResponse response = HttpResponse .ok(datasetFile.getStreamedFile()) .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + datasetFile.getFileName()); diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 0b3dfa596..89cd782ee 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -155,12 +155,14 @@ private void addObsVarDataToRow( } } private List> addBrAPIObsToRecords( - List> maps, List dataset, BrAPITrial experiment, Program program, boolean includeTimestamp, List obsVars) throws ApiException, DoesNotExistException { + // lookup table for export rows by OU id + Map> rowByOUId = new HashMap<>(); + // cache studies belonging to dataset Map studyByDbId = new HashMap<>(); for (BrAPIObservation obs: dataset) { @@ -189,11 +191,8 @@ private List> addBrAPIObsToRecords( String ouId = ouXref.getReferenceID(); // if there is a row with that ouId then just add the obs var data and timestamp to the row - Optional> existingRow = maps.stream() - .filter(row -> ouId.equals(row.get(ExperimentObservation.Columns.OBS_UNIT_ID))) - .findAny(); - if (existingRow.isPresent()) { - addObsVarDataToRow(existingRow.get(), obs, includeTimestamp, obsVars, program); + if (rowByOUId.get(ouId) != null) { + addObsVarDataToRow(rowByOUId.get(ouId), obs, includeTimestamp, obsVars, program); } else { // otherwise, make a new row @@ -201,7 +200,7 @@ private List> addBrAPIObsToRecords( BrAPIGermplasm germplasm = germplasmDAO.getGermplasmByDBID(obs.getGermplasmDbId(), program.getId()) .orElseThrow(() -> new DoesNotExistException("Germplasm not returned from BrAPI service")); BrAPIStudy study = studyByDbId.get(obs.getStudyDbId()); - row.put(ExperimentObservation.Columns.GERMPLASM_NAME, Utilities.removeProgramKey(obs.getGermplasmName(), program.getKey())); + row.put(ExperimentObservation.Columns.GERMPLASM_NAME, Utilities.removeProgramKey(obs.getGermplasmName(), program.getKey(), germplasm.getAccessionNumber())); row.put(ExperimentObservation.Columns.GERMPLASM_GID, germplasm.getAccessionNumber()); row.put(ExperimentObservation.Columns.TEST_CHECK, ou.getObservationUnitPosition().getEntryType().toString()); row.put(ExperimentObservation.Columns.EXP_TITLE, Utilities.removeProgramKey(experiment.getTrialName(), program.getKey())); @@ -209,9 +208,9 @@ private List> addBrAPIObsToRecords( row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); row.put(ExperimentObservation.Columns.ENV, obs.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.STUDY_NAME).getAsString()); - row.put(ExperimentObservation.Columns.ENV_LOCATION, study.getLocationName()); + row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); row.put(ExperimentObservation.Columns.ENV_YEAR, obs.getSeason().getYear()); - row.put(ExperimentObservation.Columns.EXP_UNIT_ID, obs.getObservationUnitName()); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKey(obs.getObservationUnitName(), program.getKey())); // get replicate number Optional repLevel = ou.getObservationUnitPosition() @@ -244,10 +243,12 @@ private List> addBrAPIObsToRecords( row.put(ExperimentObservation.Columns.OBS_UNIT_ID, ouId); addObsVarDataToRow(row, obs, includeTimestamp, obsVars, program); - maps.add(row); + rowByOUId.put(ouId, row); } } - return maps; + + // return a list of export rows + return rowByOUId.entrySet().stream().map(entry -> entry.getValue()).collect(Collectors.toList()); } private List addObsVarColumns( @@ -319,10 +320,8 @@ public DownloadFile exportObservations( } StreamedFile downloadFile; - List> experimentObservationRecords = new ArrayList<>(); - - experimentObservationRecords = addBrAPIObsToRecords(experimentObservationRecords, dataset, experiment, program, params.isIncludeTimestamps(), obsVars); + List> experimentObservationRecords = addBrAPIObsToRecords(dataset, experiment, program, params.isIncludeTimestamps(), obsVars); if (fileType.equals(FileType.CSV)){ downloadFile = CSVWriter.writeToDownload(columns, experimentObservationRecords, fileType); From 7f29b3acc5142ed632040201eaa6b92e2713de55 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 26 May 2023 10:41:33 -0400 Subject: [PATCH 12/27] update --- .../brapi/v2/ExperimentController.java | 2 +- .../brapi/v2/services/BrAPITrialService.java | 64 ++++++++++++++++++- .../ExperimentControllerIntegrationTest.java | 2 +- 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index b3af6676b..b679c653f 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -111,7 +111,7 @@ public HttpResponse datasetExport( try { Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program does not exist")); DownloadFile datasetFile = experimentService.exportObservations(program, experimentId, queryParams); - Table table = FileUtil.parseTableFromCsv(datasetFile.getStreamedFile().getInputStream()); + //Table table = FileUtil.parseTableFromCsv(datasetFile.getStreamedFile().getInputStream()); HttpResponse response = HttpResponse .ok(datasetFile.getStreamedFile()) .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + datasetFile.getFileName()); diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 89cd782ee..27ccd9e1e 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -250,6 +250,65 @@ private List> addBrAPIObsToRecords( // return a list of export rows return rowByOUId.entrySet().stream().map(entry -> entry.getValue()).collect(Collectors.toList()); } +/* + private Map createExportRow( + BrAPITrial experiment, + Program program, + BrAPIObservationUnit ou, + Map studyByDbId) throws ApiException, DoesNotExistException { + HashMap row = new HashMap<>(); + BrAPIGermplasm germplasm = germplasmDAO.getGermplasmByDBID(ou.getGermplasmDbId(), program.getId()) + .orElseThrow(() -> new DoesNotExistException("Germplasm not returned from BrAPI service")); + BrAPIStudy study = studyByDbId.get(ou.getStudyDbId()); + study.getSeasons().isEmpty() ? null : study.getSeasons().get(0) + row.put(ExperimentObservation.Columns.GERMPLASM_NAME, Utilities.removeProgramKey(ou.getGermplasmName(), program.getKey(), germplasm.getAccessionNumber())); + row.put(ExperimentObservation.Columns.GERMPLASM_GID, germplasm.getAccessionNumber()); + row.put(ExperimentObservation.Columns.TEST_CHECK, ou.getObservationUnitPosition().getEntryType().toString()); + row.put(ExperimentObservation.Columns.EXP_TITLE, Utilities.removeProgramKey(experiment.getTrialName(), program.getKey())); + row.put(ExperimentObservation.Columns.EXP_DESCRIPTION, experiment.getTrialDescription()); + row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); + row.put(ExperimentObservation.Columns.ENV, study.getStudyName()); + row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); + row.put(ExperimentObservation.Columns.ENV_YEAR, study.getSeasons() obs.getSeason().getYear()); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKey(ou.getObservationUnitName(), program.getKey())); + + // get replicate number + Optional repLevel = ou.getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(level -> BrAPIConstants.REPLICATE.getValue().equals(level.getLevelName())) + .findFirst(); + if (repLevel.isPresent()) { + row.put(ExperimentObservation.Columns.REP_NUM, Integer.parseInt(repLevel.get().getLevelCode())); + } + + //get block number + Optional blockLevel = ou.getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(level -> BrAPIConstants.BLOCK.getValue().equals(level.getLevelName())) + .findFirst(); + if (blockLevel.isPresent()) { + row.put(ExperimentObservation.Columns.BLOCK_NUM, Integer.parseInt(blockLevel.get().getLevelCode())); + } + + if (ou.getObservationUnitPosition() != null) { + row.put(ExperimentObservation.Columns.ROW, Double.parseDouble(ou.getObservationUnitPosition().getPositionCoordinateX())); + row.put(ExperimentObservation.Columns.COLUMN, Double.parseDouble(ou.getObservationUnitPosition().getPositionCoordinateY())); + } + + if (ou.getTreatments() != null && !ou.getTreatments().isEmpty()) { + row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, ou.getTreatments().get(0).getFactor().toString()); + } else { + row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, null); + } + + row.put(ExperimentObservation.Columns.OBS_UNIT_ID, ouId); + addObsVarDataToRow(row, obs, includeTimestamp, obsVars, program); + + return row; + } + + */ private List addObsVarColumns( List columns, @@ -295,11 +354,10 @@ public DownloadFile exportObservations( Program program, UUID experimentId, ExperimentExportQuery params) throws IOException, DoesNotExistException, ApiException, ParsingException { - // process params - - FileType fileType = params.getFileExtension(); + List expStudies = studyDAO.getStudiesByExperimentID(experimentId, program); + // get BrAPI observations for requested environments List dataset = getObservationDataset(program, experimentId); if (params.getEnvironments() != null) { diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index 1b9bf478e..57d73b83e 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -186,7 +186,7 @@ void setup() throws Exception { Double val1 = Math.random(); Double val2 = Math.random(); row1.put(traits.get(i).getObservationVariableName(), val1); - row2.put(traits.get(i).getObservationVariableName(), val2); + //row2.put(traits.get(i).getObservationVariableName(), val2); }; rows.add(row1); From 66337c4de572f67e377657f8c3007cf812880864 Mon Sep 17 00:00:00 2001 From: timparsons Date: Fri, 26 May 2023 15:53:35 -0400 Subject: [PATCH 13/27] [BI-1465] Adding debug statements to tests --- .../ExperimentControllerIntegrationTest.java | 50 ++++++++++++------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index 57d73b83e..499ea16bb 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -55,6 +55,7 @@ import org.junit.jupiter.api.*; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import tech.tablesaw.api.ColumnType; import tech.tablesaw.api.Row; import tech.tablesaw.api.StringColumn; import tech.tablesaw.api.Table; @@ -357,14 +358,14 @@ private void checkDownloadTable( */ - List> matchingImportRows = requestedImportRows.stream().filter(importRow -> { - for (Row downloadRow : table) { - if (isMatchedRow(importRow, downloadRow)) { - return true; - }; + List> matchingImportRows = new ArrayList<>(); + + for (int rowNum = 0; rowNum < requestedImportRows.size(); rowNum++) { + Row downloadRow = table.row(rowNum); + if(isMatchedRow(requestedImportRows.get(rowNum), downloadRow)) { + matchingImportRows.add(requestedImportRows.get(rowNum)); } - return false; - }).collect(Collectors.toList()); + } assertEquals(requestedImportRows.size(),matchingImportRows.size()); // Observation units populated @@ -373,24 +374,39 @@ private void checkDownloadTable( } private boolean isMatchedRow(Map importRow, Row downloadRow) { + System.out.println("Validating row: " + downloadRow.getRowNumber()); return importRow.entrySet().stream().filter(e -> { String header = e.getKey(); List importColumns = columns.stream().filter(col -> {return header.equals(col.getValue());}).collect(Collectors.toList()); - if (importColumns.isEmpty() || importColumns.size() > 1) { + if (importColumns.size() != 1) { return false; } - if (downloadRow.getColumnType(e.getKey()).equals(Column.ColumnDataType.STRING)) { - return downloadRow.getString(e.getKey()).equals(e.getValue()); + Object expectedVal = null; + Object downloadedVal = null; + boolean doCompare = false; + + if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.STRING)) { + expectedVal = e.getValue(); + downloadedVal = downloadRow.getString(e.getKey()); + doCompare = true; } - if (downloadRow.getColumnType(e.getKey()).equals(Column.ColumnDataType.INTEGER)) { - return downloadRow.getInt(e.getKey()) == Integer.parseInt(e.getValue().toString()); + if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.INTEGER)) { + expectedVal = Integer.parseInt(e.getValue().toString()); + downloadedVal = downloadRow.getInt(e.getKey()); + doCompare = true; } - if (downloadRow.getColumnType(e.getKey()).equals(Column.ColumnDataType.DOUBLE)) { - return downloadRow.getDouble(e.getKey()) == Double.parseDouble(e.getValue().toString()); + if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.DOUBLE)) { + expectedVal = Double.parseDouble(e.getValue().toString()); + downloadedVal = downloadRow.getDouble(e.getKey()); + doCompare = true; } - - return false; - }).collect(Collectors.toList()).size() == importRow.size(); + System.out.println("Column: "+e.getKey()+", Expected: '"+ expectedVal +"', Received: '" + downloadedVal+"'"); + if(doCompare) { + return expectedVal.equals(downloadedVal); + } else { + return false; + } + }).count() == importRow.size(); } /* From fa6aadde49220bc2d81f34ce4f475f3efd1c2f35 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 30 May 2023 16:06:46 -0400 Subject: [PATCH 14/27] add createExportRow --- .../brapi/v2/services/BrAPITrialService.java | 103 +++++++++--------- 1 file changed, 54 insertions(+), 49 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 27ccd9e1e..9db93aa66 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -129,6 +129,48 @@ public BrAPITrial getExperiment(Program program, UUID experimentId) throws ApiEx return experiments.get(0); } + public DownloadFile exportObservations( + Program program, + UUID experimentId, + ExperimentExportQuery params) throws IOException, DoesNotExistException, ApiException, ParsingException { + FileType fileType = params.getFileExtension(); + + List expStudies = studyDAO.getStudiesByExperimentID(experimentId, program); + + // get BrAPI observations for requested environments + List dataset = getObservationDataset(program, experimentId); + if (params.getEnvironments() != null) { + List envNames = new ArrayList<>(Arrays.asList(params.getEnvironments().split(","))); + dataset = filterDatasetByEnvironment(dataset, envNames); + } + + // add columns in the export for any observation variables + BrAPITrial experiment = getExperiment(program, experimentId); + List obsVars = new ArrayList<>(); + List columns = ExperimentFileColumns.getOrderedColumns(); + if (experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null) { + String obsDatasetId = experiment + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); + obsVars = getDatasetObsVars(obsDatasetId, program); + columns = addObsVarColumns(columns, obsVars, params.isIncludeTimestamps(), program); + } + + StreamedFile downloadFile; + + List> experimentObservationRecords = addBrAPIObsToRecords(dataset, experiment, program, params.isIncludeTimestamps(), obsVars); + + if (fileType.equals(FileType.CSV)){ + downloadFile = CSVWriter.writeToDownload(columns, experimentObservationRecords, fileType); + } else { + downloadFile = ExcelWriter.writeToDownload("Dataset Export", columns, experimentObservationRecords, fileType); + } + + String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); + String fileName = makeFileName(experiment, program, envFilenameFragment) + fileType.getExtension(); + return new DownloadFile(fileName, downloadFile); + } + private void addObsVarDataToRow( Map row, BrAPIObservation obs, @@ -250,17 +292,25 @@ private List> addBrAPIObsToRecords( // return a list of export rows return rowByOUId.entrySet().stream().map(entry -> entry.getValue()).collect(Collectors.toList()); } -/* + private Map createExportRow( BrAPITrial experiment, Program program, BrAPIObservationUnit ou, Map studyByDbId) throws ApiException, DoesNotExistException { HashMap row = new HashMap<>(); + + // get OU id, germplasm, and study + BrAPIExternalReference ouXref = Utilities.getExternalReference( + ou.getExternalReferences(), + String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName())) + .orElseThrow(() -> new RuntimeException("observation unit id not found")); + String ouId = ouXref.getReferenceID(); BrAPIGermplasm germplasm = germplasmDAO.getGermplasmByDBID(ou.getGermplasmDbId(), program.getId()) .orElseThrow(() -> new DoesNotExistException("Germplasm not returned from BrAPI service")); BrAPIStudy study = studyByDbId.get(ou.getStudyDbId()); - study.getSeasons().isEmpty() ? null : study.getSeasons().get(0) + + // make export row from BrAPI objects row.put(ExperimentObservation.Columns.GERMPLASM_NAME, Utilities.removeProgramKey(ou.getGermplasmName(), program.getKey(), germplasm.getAccessionNumber())); row.put(ExperimentObservation.Columns.GERMPLASM_GID, germplasm.getAccessionNumber()); row.put(ExperimentObservation.Columns.TEST_CHECK, ou.getObservationUnitPosition().getEntryType().toString()); @@ -270,7 +320,7 @@ private Map createExportRow( row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); row.put(ExperimentObservation.Columns.ENV, study.getStudyName()); row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); - row.put(ExperimentObservation.Columns.ENV_YEAR, study.getSeasons() obs.getSeason().getYear()); + row.put(ExperimentObservation.Columns.ENV_YEAR, study.getSeasons().get(0)); row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKey(ou.getObservationUnitName(), program.getKey())); // get replicate number @@ -290,25 +340,21 @@ private Map createExportRow( if (blockLevel.isPresent()) { row.put(ExperimentObservation.Columns.BLOCK_NUM, Integer.parseInt(blockLevel.get().getLevelCode())); } - if (ou.getObservationUnitPosition() != null) { row.put(ExperimentObservation.Columns.ROW, Double.parseDouble(ou.getObservationUnitPosition().getPositionCoordinateX())); row.put(ExperimentObservation.Columns.COLUMN, Double.parseDouble(ou.getObservationUnitPosition().getPositionCoordinateY())); } - if (ou.getTreatments() != null && !ou.getTreatments().isEmpty()) { row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, ou.getTreatments().get(0).getFactor().toString()); } else { row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, null); } - row.put(ExperimentObservation.Columns.OBS_UNIT_ID, ouId); - addObsVarDataToRow(row, obs, includeTimestamp, obsVars, program); return row; } - */ + private List addObsVarColumns( List columns, @@ -350,47 +396,6 @@ private List filterDatasetByEnvironment(List obs.getAdditionalInfo().getAsJsonObject() .get(BrAPIAdditionalInfoFields.STUDY_NAME).getAsString())).collect(Collectors.toList()); } - public DownloadFile exportObservations( - Program program, - UUID experimentId, - ExperimentExportQuery params) throws IOException, DoesNotExistException, ApiException, ParsingException { - FileType fileType = params.getFileExtension(); - - List expStudies = studyDAO.getStudiesByExperimentID(experimentId, program); - - // get BrAPI observations for requested environments - List dataset = getObservationDataset(program, experimentId); - if (params.getEnvironments() != null) { - List envNames = new ArrayList<>(Arrays.asList(params.getEnvironments().split(","))); - dataset = filterDatasetByEnvironment(dataset, envNames); - } - - // add columns in the export for any observation variables - BrAPITrial experiment = getExperiment(program, experimentId); - List obsVars = new ArrayList<>(); - List columns = ExperimentFileColumns.getOrderedColumns(); - if (experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null) { - String obsDatasetId = experiment - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); - obsVars = getDatasetObsVars(obsDatasetId, program); - columns = addObsVarColumns(columns, obsVars, params.isIncludeTimestamps(), program); - } - - StreamedFile downloadFile; - - List> experimentObservationRecords = addBrAPIObsToRecords(dataset, experiment, program, params.isIncludeTimestamps(), obsVars); - - if (fileType.equals(FileType.CSV)){ - downloadFile = CSVWriter.writeToDownload(columns, experimentObservationRecords, fileType); - } else { - downloadFile = ExcelWriter.writeToDownload("Dataset Export", columns, experimentObservationRecords, fileType); - } - - String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); - String fileName = makeFileName(experiment, program, envFilenameFragment) + fileType.getExtension(); - return new DownloadFile(fileName, downloadFile); - } private Map makeExpObsMapByHeader(ExperimentObservation obs) { Map row = new HashMap<>(); From 2502ac77cd11f4a8b205d19f5e9552ea43156401 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 31 May 2023 14:48:11 -0400 Subject: [PATCH 15/27] temp fix --- .../brapi/v2/services/BrAPITrialService.java | 80 +++++++++++++------ 1 file changed, 55 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 9db93aa66..99fa2b695 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -135,8 +135,16 @@ public DownloadFile exportObservations( ExperimentExportQuery params) throws IOException, DoesNotExistException, ApiException, ParsingException { FileType fileType = params.getFileExtension(); + // get selected environments for the experiment + // get the OUs for the selected studies + // make export columns including columns for requested dataset obsvars and timestamps if requested + // make export rows from any observations + // make export rows for OUs without observations + List expStudies = studyDAO.getStudiesByExperimentID(experimentId, program); + + // get BrAPI observations for requested environments List dataset = getObservationDataset(program, experimentId); if (params.getEnvironments() != null) { @@ -171,31 +179,6 @@ public DownloadFile exportObservations( return new DownloadFile(fileName, downloadFile); } - private void addObsVarDataToRow( - Map row, - BrAPIObservation obs, - boolean includeTimestamp, - List obsVars, - Program program) { - // get observation variable for BrAPI observation - BrAPIObservationVariable var = obsVars.stream() - .filter(obsVar -> obs.getObservationVariableName().equals(obsVar.getObservationVariableName())) - .collect(Collectors.toList()).get(0); - - String varName = Utilities.removeProgramKey(obs.getObservationVariableName(), program.getKey()); - if (var.getScale().getDataType().equals(BrAPITraitDataType.ORDINAL)) { - row.put(varName, Integer.parseInt(obs.getValue())); - } else if (var.getScale().getDataType().equals(BrAPITraitDataType.NUMERICAL) || - var.getScale().getDataType().equals(BrAPITraitDataType.DURATION)) { - row.put(varName, Double.parseDouble(obs.getValue())); - } else { - row.put(varName, obs.getValue()); - } - - if (includeTimestamp) { - row.put(String.format("TS:%s",varName), obs.getObservationTimeStamp()); - } - } private List> addBrAPIObsToRecords( List dataset, BrAPITrial experiment, @@ -293,6 +276,53 @@ private List> addBrAPIObsToRecords( return rowByOUId.entrySet().stream().map(entry -> entry.getValue()).collect(Collectors.toList()); } + private void addObsVarDataToRow( + Map row, + BrAPIObservation obs, + boolean includeTimestamp, + List obsVars, + Program program) { + // get observation variable for BrAPI observation + BrAPIObservationVariable var = obsVars.stream() + .filter(obsVar -> obs.getObservationVariableName().equals(obsVar.getObservationVariableName())) + .collect(Collectors.toList()).get(0); + + String varName = Utilities.removeProgramKey(obs.getObservationVariableName(), program.getKey()); + if (var.getScale().getDataType().equals(BrAPITraitDataType.ORDINAL)) { + row.put(varName, Integer.parseInt(obs.getValue())); + } else if (var.getScale().getDataType().equals(BrAPITraitDataType.NUMERICAL) || + var.getScale().getDataType().equals(BrAPITraitDataType.DURATION)) { + row.put(varName, Double.parseDouble(obs.getValue())); + } else { + row.put(varName, obs.getValue()); + } + + if (includeTimestamp) { + row.put(String.format("TS:%s",varName), obs.getObservationTimeStamp()); + } + } + + public List getDatasetObsVars(String datasetId, Program program) throws ApiException, DoesNotExistException { + List lists = listDAO.getListByTypeAndExternalRef( + BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", referenceSource, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(datasetId)); + if (lists == null || lists.isEmpty()) { + throw new DoesNotExistException("Dataset observation variables list not returned from BrAPI service"); + } + String listDbId = lists.get(0).getListDbId(); + BrAPIListsSingleResponse list = listDAO.getListById(listDbId, program.getId()); + List obsVarNames = list.getResult().getData(); + List obsVars = obsVarDAO.getVariableByName(obsVarNames, program.getId()); + return obsVars; + } + + public BrAPITrial getExperiment(Program program, UUID experimentId) throws ApiException { + List experiments = trialDAO.getTrialsByExperimentIds(List.of(experimentId), program); + return experiments.get(0); + } + private Map createExportRow( BrAPITrial experiment, Program program, From 8b30a1155a3298074a9aaa393e5c168c331954f9 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 2 Jun 2023 14:48:14 -0400 Subject: [PATCH 16/27] handle export of empty datasets --- .../brapi/v2/services/BrAPITrialService.java | 99 +++++++++++++------ 1 file changed, 71 insertions(+), 28 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 99fa2b695..c87474db0 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -2,6 +2,7 @@ import io.micronaut.http.server.exceptions.InternalServerException; import io.micronaut.http.server.types.files.StreamedFile; +import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.BrAPIExternalReference; @@ -133,45 +134,81 @@ public DownloadFile exportObservations( Program program, UUID experimentId, ExperimentExportQuery params) throws IOException, DoesNotExistException, ApiException, ParsingException { + StreamedFile downloadFile; + boolean isDataset = false; + List dataset = new ArrayList<>(); + List obsVars = new ArrayList<>(); + Map> rowByOUId = new HashMap<>(); + Map studyByDbId = new HashMap<>(); + List requestedEnvNames = StringUtils.isNotBlank(params.getEnvironments()) ? + new ArrayList<>(Arrays.asList(params.getEnvironments().split(","))) : new ArrayList<>(); FileType fileType = params.getFileExtension(); - // get selected environments for the experiment - // get the OUs for the selected studies - // make export columns including columns for requested dataset obsvars and timestamps if requested - // make export rows from any observations - // make export rows for OUs without observations - + // get requested environments for the experiment List expStudies = studyDAO.getStudiesByExperimentID(experimentId, program); + if (!requestedEnvNames.isEmpty()) { + expStudies = expStudies.stream().filter(study -> { + return requestedEnvNames.contains(study.getStudyName()); + }).collect(Collectors.toList()); + } + expStudies.stream().forEach(study -> { + if (studyByDbId.get(study.getStudyDbId()) == null) { + studyByDbId.put(study.getStudyDbId(), study); + } + }); + // get the OUs for the requested environments + List ous = new ArrayList<>(); + try { + for (BrAPIStudy study: expStudies) { + ous.addAll(ouDAO.getObservationUnitsForStudyDbId(study.getStudyDbId(), program)); + } + } catch (ApiException err) { - - // get BrAPI observations for requested environments - List dataset = getObservationDataset(program, experimentId); - if (params.getEnvironments() != null) { - List envNames = new ArrayList<>(Arrays.asList(params.getEnvironments().split(","))); - dataset = filterDatasetByEnvironment(dataset, envNames); } - // add columns in the export for any observation variables - BrAPITrial experiment = getExperiment(program, experimentId); - List obsVars = new ArrayList<>(); + // make export columns including columns for requested dataset obsvars and timestamps if requested List columns = ExperimentFileColumns.getOrderedColumns(); - if (experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null) { + BrAPITrial experiment = getExperiment(program, experimentId); + if ((StringUtils.isBlank(params.getDataset()) || "observations".equals(params.getDataset())) && + experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null) { String obsDatasetId = experiment .getAdditionalInfo().getAsJsonObject() .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); + isDataset = true; obsVars = getDatasetObsVars(obsDatasetId, program); columns = addObsVarColumns(columns, obsVars, params.isIncludeTimestamps(), program); + } - StreamedFile downloadFile; + // make export rows from any observations + if (isDataset) { + dataset = observationDAO.getObservationsByTrialDbId(List.of(experiment.getTrialDbId()), program); + } + if (!requestedEnvNames.isEmpty()) { + dataset = filterDatasetByEnvironment(dataset, requestedEnvNames); + } + rowByOUId = addBrAPIObsToRecords(dataset, experiment, program, rowByOUId, params.isIncludeTimestamps(), obsVars); - List> experimentObservationRecords = addBrAPIObsToRecords(dataset, experiment, program, params.isIncludeTimestamps(), obsVars); + // make export rows for OUs without observations + if (rowByOUId.size() < ous.size()) { + for (BrAPIObservationUnit ou: ous) { + String ouId = getOUId(ou); + if (!rowByOUId.containsKey(ouId)) { + rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId)); + } + } + } + // write export data to requested file format + List> exportRows = rowByOUId + .entrySet() + .stream() + .map(entry -> entry.getValue()).collect(Collectors.toList()); if (fileType.equals(FileType.CSV)){ - downloadFile = CSVWriter.writeToDownload(columns, experimentObservationRecords, fileType); + downloadFile = CSVWriter.writeToDownload(columns, exportRows, fileType); } else { - downloadFile = ExcelWriter.writeToDownload("Dataset Export", columns, experimentObservationRecords, fileType); + downloadFile = ExcelWriter.writeToDownload("Dataset Export", columns, exportRows, fileType); } String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); @@ -179,14 +216,15 @@ public DownloadFile exportObservations( return new DownloadFile(fileName, downloadFile); } - private List> addBrAPIObsToRecords( + private Map> addBrAPIObsToRecords( List dataset, BrAPITrial experiment, Program program, + Map> rowByOUId, boolean includeTimestamp, List obsVars) throws ApiException, DoesNotExistException { // lookup table for export rows by OU id - Map> rowByOUId = new HashMap<>(); + // Map> rowByOUId = new HashMap<>(); // cache studies belonging to dataset Map studyByDbId = new HashMap<>(); @@ -209,11 +247,7 @@ private List> addBrAPIObsToRecords( .filter(unit -> obs.getObservationUnitDbId().equals(unit.getObservationUnitDbId())) .findAny() .orElseThrow(() -> new RuntimeException()); - BrAPIExternalReference ouXref = Utilities.getExternalReference( - ou.getExternalReferences(), - String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName())) - .orElseThrow(() -> new RuntimeException("observation unit id not found")); - String ouId = ouXref.getReferenceID(); + String ouId = getOUId(ou); // if there is a row with that ouId then just add the obs var data and timestamp to the row if (rowByOUId.get(ouId) != null) { @@ -273,7 +307,16 @@ private List> addBrAPIObsToRecords( } // return a list of export rows - return rowByOUId.entrySet().stream().map(entry -> entry.getValue()).collect(Collectors.toList()); + return rowByOUId; + //return rowByOUId.entrySet().stream().map(entry -> entry.getValue()).collect(Collectors.toList()); + } + + private String getOUId(BrAPIObservationUnit ou) { + BrAPIExternalReference ouXref = Utilities.getExternalReference( + ou.getExternalReferences(), + String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName())) + .orElseThrow(() -> new RuntimeException("observation unit id not found")); + return ouXref.getReferenceID(); } private void addObsVarDataToRow( From 76abca4399a5e91bc836fd24915502b0865bc513 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 2 Jun 2023 15:36:21 -0400 Subject: [PATCH 17/27] add null checks --- .../brapi/v2/services/BrAPITrialService.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index c87474db0..f73321739 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -386,7 +386,8 @@ private Map createExportRow( // make export row from BrAPI objects row.put(ExperimentObservation.Columns.GERMPLASM_NAME, Utilities.removeProgramKey(ou.getGermplasmName(), program.getKey(), germplasm.getAccessionNumber())); row.put(ExperimentObservation.Columns.GERMPLASM_GID, germplasm.getAccessionNumber()); - row.put(ExperimentObservation.Columns.TEST_CHECK, ou.getObservationUnitPosition().getEntryType().toString()); + String testCheck = ou.getObservationUnitPosition().getEntryType() != null ? ou.getObservationUnitPosition().getEntryType().toString() : null; + row.put(ExperimentObservation.Columns.TEST_CHECK, testCheck); row.put(ExperimentObservation.Columns.EXP_TITLE, Utilities.removeProgramKey(experiment.getTrialName(), program.getKey())); row.put(ExperimentObservation.Columns.EXP_DESCRIPTION, experiment.getTrialDescription()); row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); @@ -413,7 +414,8 @@ private Map createExportRow( if (blockLevel.isPresent()) { row.put(ExperimentObservation.Columns.BLOCK_NUM, Integer.parseInt(blockLevel.get().getLevelCode())); } - if (ou.getObservationUnitPosition() != null) { + if (ou.getObservationUnitPosition() != null && ou.getObservationUnitPosition().getPositionCoordinateX() != null && + ou.getObservationUnitPosition().getPositionCoordinateY() != null) { row.put(ExperimentObservation.Columns.ROW, Double.parseDouble(ou.getObservationUnitPosition().getPositionCoordinateX())); row.put(ExperimentObservation.Columns.COLUMN, Double.parseDouble(ou.getObservationUnitPosition().getPositionCoordinateY())); } From e44458bd60b12f932f5e4fd679279c0a128ca757 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 5 Jun 2023 16:46:32 -0400 Subject: [PATCH 18/27] remove keys from env and exp unit id --- .../brapi/v2/services/BrAPITrialService.java | 34 ++++++++--- .../ExperimentControllerIntegrationTest.java | 61 ++++++++++++------- 2 files changed, 63 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index f73321739..0f5866a3e 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -15,6 +15,7 @@ import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.Utilities; +import org.brapi.v2.model.core.*; import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.*; @@ -61,6 +62,7 @@ public class BrAPITrialService { private final BrAPIListDAO listDAO; private final BrAPIObservationVariableDAO obsVarDAO; private final BrAPIStudyDAO studyDAO; + private final BrAPISeasonDAO seasonDAO; private final BrAPIObservationUnitDAO ouDAO; private final BrAPIGermplasmDAO germplasmDAO; @@ -72,6 +74,7 @@ public BrAPITrialService(ProgramService programService, BrAPIListDAO listDAO, BrAPIObservationVariableDAO obsVarDAO, BrAPIStudyDAO studyDAO, + BrAPISeasonDAO seasonDAO, BrAPIObservationUnitDAO ouDAO, BrAPIGermplasmDAO germplasmDAO) { this.programService = programService; @@ -81,6 +84,7 @@ public BrAPITrialService(ProgramService programService, this.listDAO = listDAO; this.obsVarDAO = obsVarDAO; this.studyDAO = studyDAO; + this.seasonDAO = seasonDAO; this.ouDAO = ouDAO; this.germplasmDAO = germplasmDAO; } @@ -170,14 +174,14 @@ public DownloadFile exportObservations( // make export columns including columns for requested dataset obsvars and timestamps if requested List columns = ExperimentFileColumns.getOrderedColumns(); BrAPITrial experiment = getExperiment(program, experimentId); - if ((StringUtils.isBlank(params.getDataset()) || "observations".equals(params.getDataset())) && + if ((StringUtils.isBlank(params.getDataset()) || "observations".equalsIgnoreCase(params.getDataset())) && experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null) { String obsDatasetId = experiment .getAdditionalInfo().getAsJsonObject() .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); isDataset = true; obsVars = getDatasetObsVars(obsDatasetId, program); - columns = addObsVarColumns(columns, obsVars, params.isIncludeTimestamps(), program); + addObsVarColumns(columns, obsVars, params.isIncludeTimestamps(), program); } @@ -188,7 +192,7 @@ public DownloadFile exportObservations( if (!requestedEnvNames.isEmpty()) { dataset = filterDatasetByEnvironment(dataset, requestedEnvNames); } - rowByOUId = addBrAPIObsToRecords(dataset, experiment, program, rowByOUId, params.isIncludeTimestamps(), obsVars); + addBrAPIObsToRecords(dataset, experiment, program, rowByOUId, params.isIncludeTimestamps(), obsVars); // make export rows for OUs without observations if (rowByOUId.size() < ous.size()) { @@ -261,15 +265,21 @@ private Map> addBrAPIObsToRecords( BrAPIStudy study = studyByDbId.get(obs.getStudyDbId()); row.put(ExperimentObservation.Columns.GERMPLASM_NAME, Utilities.removeProgramKey(obs.getGermplasmName(), program.getKey(), germplasm.getAccessionNumber())); row.put(ExperimentObservation.Columns.GERMPLASM_GID, germplasm.getAccessionNumber()); - row.put(ExperimentObservation.Columns.TEST_CHECK, ou.getObservationUnitPosition().getEntryType().toString()); + + // use only the capitalized first character of the entry type for test/check + BrAPIEntryTypeEnum entryType = ou.getObservationUnitPosition().getEntryType(); + String testCheck = entryType != null ? String.valueOf(Character.toUpperCase(entryType.toString().charAt(0))) : null; + row.put(ExperimentObservation.Columns.TEST_CHECK, testCheck); row.put(ExperimentObservation.Columns.EXP_TITLE, Utilities.removeProgramKey(experiment.getTrialName(), program.getKey())); row.put(ExperimentObservation.Columns.EXP_DESCRIPTION, experiment.getTrialDescription()); row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); - row.put(ExperimentObservation.Columns.ENV, obs.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.STUDY_NAME).getAsString()); + String keyedEnvName = obs.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.STUDY_NAME).getAsString(); + String envName = Utilities.removeProgramKeyAndUnknownAdditionalData(keyedEnvName, program.getKey()); + row.put(ExperimentObservation.Columns.ENV, envName); row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); row.put(ExperimentObservation.Columns.ENV_YEAR, obs.getSeason().getYear()); - row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKey(obs.getObservationUnitName(), program.getKey())); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getObservationUnitName(), program.getKey())); // get replicate number Optional repLevel = ou.getObservationUnitPosition() @@ -386,16 +396,20 @@ private Map createExportRow( // make export row from BrAPI objects row.put(ExperimentObservation.Columns.GERMPLASM_NAME, Utilities.removeProgramKey(ou.getGermplasmName(), program.getKey(), germplasm.getAccessionNumber())); row.put(ExperimentObservation.Columns.GERMPLASM_GID, germplasm.getAccessionNumber()); - String testCheck = ou.getObservationUnitPosition().getEntryType() != null ? ou.getObservationUnitPosition().getEntryType().toString() : null; + + // use only the capitalized first character of the entry type for test/check + BrAPIEntryTypeEnum entryType = ou.getObservationUnitPosition().getEntryType(); + String testCheck = entryType != null ? String.valueOf(Character.toUpperCase(entryType.toString().charAt(0))) : null; row.put(ExperimentObservation.Columns.TEST_CHECK, testCheck); row.put(ExperimentObservation.Columns.EXP_TITLE, Utilities.removeProgramKey(experiment.getTrialName(), program.getKey())); row.put(ExperimentObservation.Columns.EXP_DESCRIPTION, experiment.getTrialDescription()); row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); - row.put(ExperimentObservation.Columns.ENV, study.getStudyName()); + row.put(ExperimentObservation.Columns.ENV, Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); - row.put(ExperimentObservation.Columns.ENV_YEAR, study.getSeasons().get(0)); - row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKey(ou.getObservationUnitName(), program.getKey())); + BrAPISeason season = seasonDAO.getSeasonById(study.getSeasons().get(0), program.getId()); + row.put(ExperimentObservation.Columns.ENV_YEAR, season.getYear()); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); // get replicate number Optional repLevel = ou.getObservationUnitPosition() diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index 499ea16bb..e1b3f4663 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -184,8 +184,14 @@ void setup() throws Exception { // Add test observation data for (int i = 0; i < traits.size(); i++) { - Double val1 = Math.random(); - Double val2 = Math.random(); + Random random = new Random(); + + // TODO: test for sending obs data as double + // Double val1 = Math.random(); + // Double val2 = Math.random(); + + Float val1 = random.nextFloat(); + Float val2 = random.nextFloat(); row1.put(traits.get(i).getObservationVariableName(), val1); //row2.put(traits.get(i).getObservationVariableName(), val2); }; @@ -309,7 +315,8 @@ private void checkDownloadTable( String requestedEnv, List> importRows, Table table, - boolean includeTimestamps) { + boolean includeTimestamps, + String extension) { // Filename is correct: _Observation Dataset [-]__ List> requestedImportRows; @@ -344,37 +351,41 @@ private void checkDownloadTable( } - // All requested import data included in download - // update columns with traitName to use "traitName [progrmKey]" - /* - List traitNames = traits.stream().map(trait -> trait.getObservationVariableName()).collect(Collectors.toList()); - for (Map row: requestedImportRows) { - row.entrySet().stream().forEach(column -> { - if (traitNames.contains(column.getKey())) { - - } - }); - } - - */ - List> matchingImportRows = new ArrayList<>(); + Optional> matchingImportRow; for (int rowNum = 0; rowNum < requestedImportRows.size(); rowNum++) { Row downloadRow = table.row(rowNum); - if(isMatchedRow(requestedImportRows.get(rowNum), downloadRow)) { - matchingImportRows.add(requestedImportRows.get(rowNum)); + + // sort order is not guaranteed to be th same as import, so find import row for corresponding export row + // by first matching environment and GID + matchingImportRow = requestedImportRows.stream().filter(row -> { + String gid = ExperimentObservation.Columns.GERMPLASM_GID; + String env = ExperimentObservation.Columns.ENV; + if (extension.equalsIgnoreCase(FileType.CSV.getName())) { + return Integer.parseInt(row.get(gid).toString()) == downloadRow.getInt(gid) && + row.get(env).equals(downloadRow.getString(env)); + } else { + return row.get(gid).equals(downloadRow.getString(gid)) && row.get(env).equals(downloadRow.getString(env)); + } + }).findAny(); + assertTrue(matchingImportRow.isPresent()); + + // then check the rest of the fields match + if (isMatchedRow(matchingImportRow.get(), downloadRow)) { + matchingImportRows.add(matchingImportRow.get()); } } assertEquals(requestedImportRows.size(),matchingImportRows.size()); // Observation units populated assertEquals(0, table.column("ObsUnitID").countMissing()); - assertEquals(importRows.size(), table.column("ObsUnitID").countUnique()); + assertEquals(requestedImportRows.size(), table.column("ObsUnitID").countUnique()); } private boolean isMatchedRow(Map importRow, Row downloadRow) { System.out.println("Validating row: " + downloadRow.getRowNumber()); + System.out.println("import columns: " + importRow.size()); return importRow.entrySet().stream().filter(e -> { String header = e.getKey(); List importColumns = columns.stream().filter(col -> {return header.equals(col.getValue());}).collect(Collectors.toList()); @@ -386,7 +397,7 @@ private boolean isMatchedRow(Map importRow, Row downloadRow) { boolean doCompare = false; if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.STRING)) { - expectedVal = e.getValue(); + expectedVal = e.getValue().toString(); downloadedVal = downloadRow.getString(e.getKey()); doCompare = true; } @@ -400,8 +411,14 @@ private boolean isMatchedRow(Map importRow, Row downloadRow) { downloadedVal = downloadRow.getDouble(e.getKey()); doCompare = true; } + if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.FLOAT)) { + expectedVal = e.getValue(); + downloadedVal = downloadRow.getFloat(e.getKey()); + doCompare = true; + } System.out.println("Column: "+e.getKey()+", Expected: '"+ expectedVal +"', Received: '" + downloadedVal+"'"); if(doCompare) { + assertEquals(expectedVal, downloadedVal); return expectedVal.equals(downloadedVal); } else { return false; @@ -466,6 +483,6 @@ void downloadDatasets(boolean includeTimestamps, boolean hasEmptyObs, String req if (extension.equals("XLS") || extension.equals("XLSX")) { download = FileUtil.parseTableFromExcel(bodyStream, 0); } - checkDownloadTable(requestedEnv, rows, download, includeTimestamps); + checkDownloadTable(requestedEnv, rows, download, includeTimestamps, extension); } } From ff449c3808381dd53deb9e9e2a3801a0934baad7 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 5 Jun 2023 18:33:15 -0400 Subject: [PATCH 19/27] clean up test --- .../ExperimentControllerIntegrationTest.java | 233 +++++++----------- 1 file changed, 93 insertions(+), 140 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index e1b3f4663..af335d294 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -1,7 +1,6 @@ package org.breedinginsight.brapi.v2; import com.google.gson.*; -import com.ibm.icu.text.UFormat; import io.kowalski.fannypack.FannyPack; import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpResponse; @@ -9,30 +8,16 @@ import io.micronaut.http.MediaType; import io.micronaut.http.client.RxHttpClient; import io.micronaut.http.client.annotation.Client; -import io.micronaut.http.client.exceptions.HttpClientResponseException; -import io.micronaut.http.client.netty.FullNettyClientHttpResponse; import io.micronaut.http.netty.cookies.NettyCookie; import io.micronaut.test.annotation.MicronautTest; import io.reactivex.Flowable; import lombok.SneakyThrows; -import org.apache.poi.hssf.usermodel.HSSFSheet; -import org.apache.poi.hssf.usermodel.HSSFWorkbook; -import org.apache.poi.ss.usermodel.Workbook; -import org.apache.poi.ss.usermodel.WorkbookFactory; -import org.apache.poi.xssf.usermodel.XSSFWorkbook; -import org.apache.poi.xssf.usermodel.XSSFWorkbookFactory; -import org.apache.xmlbeans.impl.xb.ltgfmt.TestsDocument; import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.core.BrAPIProgram; -import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.brapi.v2.model.pheno.BrAPIObservation; -import org.brapi.v2.model.pheno.BrAPITrait; import org.breedinginsight.BrAPITest; import org.breedinginsight.TestUtils; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.model.v1.request.ProgramRequest; -import org.breedinginsight.api.model.v1.request.SharedOntologyProgramRequest; import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.api.v1.controller.TestTokenValidator; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; @@ -41,33 +26,26 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; -import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.SpeciesDAO; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.*; import org.breedinginsight.services.OntologyService; import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; import org.breedinginsight.services.writers.CSVWriter; import org.breedinginsight.utilities.FileUtil; -import org.jooq.ContentType; import org.jooq.DSLContext; -import org.jooq.tools.csv.CSVReader; import org.junit.jupiter.api.*; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import tech.tablesaw.api.ColumnType; import tech.tablesaw.api.Row; -import tech.tablesaw.api.StringColumn; import tech.tablesaw.api.Table; - import javax.inject.Inject; -import javax.validation.constraints.AssertTrue; import java.io.*; -import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; import java.util.*; import java.util.stream.Collectors; - import static io.micronaut.http.HttpRequest.*; import static org.junit.jupiter.api.Assertions.*; @@ -76,26 +54,17 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class ExperimentControllerIntegrationTest extends BrAPITest { - private FannyPack securityFp; - private FannyPack brapiFp; - private FannyPack fp; - private FannyPack brapiObservationFp; private Program program; - private ImportTestUtils importTestUtils; - private String mappingId; private String experimentId; - private List> rows = new ArrayList<>(); - private List columns = new ArrayList<>(); + private final List> rows = new ArrayList<>(); + private final List columns = ExperimentFileColumns.getOrderedColumns(); private List traits; - private final String GERMPLASM_LIST_NAME = "Program Germplasm List"; - private final String GERMPLASM_LIST_DESC = "Program Germplasm List"; + @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @Inject private DSLContext dsl; @Inject - private ProgramDAO programDAO; - @Inject private UserDAO userDAO; @Inject private SpeciesDAO speciesDAO; @@ -108,20 +77,19 @@ public class ExperimentControllerIntegrationTest extends BrAPITest { @Client("/${micronaut.bi.api.version}") private RxHttpClient client; - private Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) + private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .create(); @BeforeAll void setup() throws Exception { - fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); - importTestUtils = new ImportTestUtils(); - securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); - brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); - //brapiObservationFp = FannyPack.fill("src/test/resources/sql/brapi/BrAPIOntologyControllerIntegrationTest.sql"); + FannyPack fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); + ImportTestUtils importTestUtils = new ImportTestUtils(); + FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); // Test User - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Species @@ -153,7 +121,7 @@ void setup() throws Exception { .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class ); HttpResponse response = call.blockingFirst(); - mappingId = JsonParser.parseString(response.body()).getAsJsonObject() + String mappingId = JsonParser.parseString(Objects.requireNonNull(response.body())).getAsJsonObject() .getAsJsonObject("result") .getAsJsonArray("data") .get(0).getAsJsonObject().get("id").getAsString(); @@ -183,24 +151,28 @@ void setup() throws Exception { Map row2 = makeExpImportRow("Env2"); // Add test observation data - for (int i = 0; i < traits.size(); i++) { + for (Trait trait : traits) { Random random = new Random(); - // TODO: test for sending obs data as double + // TODO: test for sending obs data as double. + // A float is returned from the backend instead of double. there is a separate card to fix this. // Double val1 = Math.random(); - // Double val2 = Math.random(); Float val1 = random.nextFloat(); - Float val2 = random.nextFloat(); - row1.put(traits.get(i).getObservationVariableName(), val1); - //row2.put(traits.get(i).getObservationVariableName(), val2); - }; + row1.put(trait.getObservationVariableName(), val1); + } rows.add(row1); rows.add(row2); // Import test experiment, environments, and any observations - JsonObject importResult = importTestUtils.uploadAndFetch(writeDataToFile(rows, traits), null, true, client, program, mappingId); + JsonObject importResult = importTestUtils.uploadAndFetch( + writeDataToFile(rows, traits), + null, + true, + client, + program, + mappingId); experimentId = importResult .get("preview").getAsJsonObject() .get("rows").getAsJsonArray() @@ -209,33 +181,74 @@ void setup() throws Exception { .get("id").getAsString(); } + /* + Tests + - export empty dataset, single environment, csv format + - export empty dataset, single environment, xls format + - export empty dataset, single environment, xlsx format + - export populated dataset, single environment, csv format + - export populated dataset, single environment, xls format + - export populated dataset, single environment, xlsx format + - export empty dataset, multiple environment, csv format + - export empty dataset, multiple environment, xls format + - export empty dataset, multiple environment, xlsx format + - export populated dataset, multiple environment, csv format + - export populated dataset, multiple environment, xls format + - export populated dataset, multiple environment, xlsx format + */ + @ParameterizedTest + @CsvSource(value = {"true,,CSV", "true,Env1,CSV", + "false,,CSV", "false,Env1,CSV", + "true,,XLS", "true,Env1,XLS", + "false,,XLS", "false,Env1,XLS", + "true,,XLSX", "true,Env1,XLSX", + "false,,XLSX", "false,Env1,XLSX",}) + @SneakyThrows + void downloadDatasets(boolean includeTimestamps, String requestedEnv, String extension) { + // Download test experiment + String envParam = "all=true"; + if (requestedEnv != null) { + envParam = "environments=" + requestedEnv; + } + Flowable> call = client.exchange( + GET(String.format("/programs/%s/experiments/%s/export?includeTimestamps=%s&%s&fileExtension=%s", + program.getId().toString(), experimentId, includeTimestamps, envParam, extension)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse response = call.blockingFirst(); + + // Assert 200 response + assertEquals(HttpStatus.OK, response.getStatus()); + + // Assert file format fidelity + Map mediaTypeByExtension = new HashMap<>(); + mediaTypeByExtension.put("CSV", FileType.CSV.getMimeType()); + mediaTypeByExtension.put("XLS", FileType.XLS.getMimeType()); + mediaTypeByExtension.put("XLSX", FileType.XLSX.getMimeType()); + String downloadMediaType = response.getHeaders().getContentType().orElseThrow(Exception::new); + assertEquals(mediaTypeByExtension.get(extension), downloadMediaType); + + // Assert import/export fidelity and presence of observation units in export + ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(response.body())); + Table download = Table.create(); + if (extension.equals("CSV")) { + download = FileUtil.parseTableFromCsv(bodyStream); + } + if (extension.equals("XLS") || extension.equals("XLSX")) { + download = FileUtil.parseTableFromExcel(bodyStream, 0); + } + checkDownloadTable(requestedEnv, rows, download, includeTimestamps, extension); + } private File writeDataToFile(List> data, List traits) throws IOException { File file = File.createTempFile("test", ".csv"); - columns.add(Column.builder().value(ExperimentObservation.Columns.GERMPLASM_NAME).dataType(Column.ColumnDataType.STRING).build()); - columns.add(Column.builder().value(ExperimentObservation.Columns.GERMPLASM_GID).dataType(Column.ColumnDataType.INTEGER).build()); - columns.add(Column.builder().value(ExperimentObservation.Columns.TEST_CHECK).dataType(Column.ColumnDataType.STRING).build()); - columns.add(Column.builder().value(ExperimentObservation.Columns.EXP_TITLE).dataType(Column.ColumnDataType.STRING).build()); - columns.add(Column.builder().value(ExperimentObservation.Columns.EXP_DESCRIPTION).dataType(Column.ColumnDataType.STRING).build()); - columns.add(Column.builder().value(ExperimentObservation.Columns.EXP_UNIT).dataType(Column.ColumnDataType.STRING).build()); - columns.add(Column.builder().value(ExperimentObservation.Columns.EXP_TYPE).dataType(Column.ColumnDataType.STRING).build()); - columns.add(Column.builder().value(ExperimentObservation.Columns.ENV).dataType(Column.ColumnDataType.STRING).build()); - columns.add(Column.builder().value(ExperimentObservation.Columns.ENV_LOCATION).dataType(Column.ColumnDataType.STRING).build()); - columns.add(Column.builder().value(ExperimentObservation.Columns.ENV_YEAR).dataType(Column.ColumnDataType.INTEGER).build()); - columns.add(Column.builder().value(ExperimentObservation.Columns.EXP_UNIT_ID).dataType(Column.ColumnDataType.STRING).build()); - columns.add(Column.builder().value(ExperimentObservation.Columns.REP_NUM).dataType(Column.ColumnDataType.INTEGER).build()); - columns.add(Column.builder().value(ExperimentObservation.Columns.BLOCK_NUM).dataType(Column.ColumnDataType.INTEGER).build()); - columns.add(Column.builder().value(ExperimentObservation.Columns.ROW).dataType(Column.ColumnDataType.INTEGER).build()); - columns.add(Column.builder().value(ExperimentObservation.Columns.COLUMN).dataType(Column.ColumnDataType.INTEGER).build()); - columns.add(Column.builder().value(ExperimentObservation.Columns.TREATMENT_FACTORS).dataType(Column.ColumnDataType.STRING).build()); - columns.add(Column.builder().value(ExperimentObservation.Columns.OBS_UNIT_ID).dataType(Column.ColumnDataType.STRING).build()); - if(traits != null) { - traits.forEach(trait -> { - columns.add(Column.builder().value(trait.getObservationVariableName()).dataType(Column.ColumnDataType.STRING).build()); - }); + traits.forEach(trait -> columns.add( + Column.builder() + .value(trait.getObservationVariableName()) + .dataType(Column.ColumnDataType.STRING) + .build())); } - ByteArrayOutputStream byteArrayOutputStream = CSVWriter.writeToCSV(columns, data); FileOutputStream fos = new FileOutputStream(file); fos.write(byteArrayOutputStream.toByteArray()); @@ -335,16 +348,14 @@ private void checkDownloadTable( .map(row -> row.get(ExperimentObservation.Columns.ENV).toString()) .distinct() .collect(Collectors.toList()) - .forEach(envName -> { - assertTrue(table.stringColumn("Env").contains(envName)); - }); + .forEach(envName -> assertTrue(table.stringColumn("Env").contains(envName))); } else { // Only requested environment downloaded - requestedImportRows = importRows.stream().filter(row -> { - return row.get("Env").toString().equals(requestedEnv); - }).collect(Collectors.toList()); + requestedImportRows = importRows + .stream() + .filter(row -> row.get("Env").toString().equals(requestedEnv)).collect(Collectors.toList()); assertEquals(1, table.stringColumn("Env").countUnique()); assertTrue(table.stringColumn("Env").contains(requestedEnv)); @@ -388,7 +399,9 @@ private boolean isMatchedRow(Map importRow, Row downloadRow) { System.out.println("import columns: " + importRow.size()); return importRow.entrySet().stream().filter(e -> { String header = e.getKey(); - List importColumns = columns.stream().filter(col -> {return header.equals(col.getValue());}).collect(Collectors.toList()); + List importColumns = columns + .stream() + .filter(col -> header.equals(col.getValue())).collect(Collectors.toList()); if (importColumns.size() != 1) { return false; } @@ -425,64 +438,4 @@ private boolean isMatchedRow(Map importRow, Row downloadRow) { } }).count() == importRow.size(); } - - /* - Tests - - export empty dataset, single environment, csv format - - export empty dataset, single environment, xls format - - export empty dataset, single environment, xlsx format - - export populated dataset, single environment, csv format - - export populated dataset, single environment, xls format - - export populated dataset, single environment, xlsx format - - export empty dataset, multiple environment, csv format - - export empty dataset, multiple environment, xls format - - export empty dataset, multiple environment, xlsx format - - export populated dataset, multiple environment, csv format - - export populated dataset, multiple environment, xls format - - export populated dataset, multiple environment, xlsx format - */ - - @ParameterizedTest - @CsvSource(value = {"true,true,,CSV", "true,false,,CSV", "true,true,Env1,CSV", "true,false,Env1,CSV", - "false,true,,CSV", "false,false,,CSV", "false,true,Env1,CSV", "false,false,Env1,CSV", - "true,true,,XLS", "true,false,,XLS", "true,true,Env1,XLS", "true,false,Env1,XLS", - "false,true,,XLS", "false,false,,XLS", "false,true,Env1,XLS", "false,false,Env1,XLS", - "true,true,,XLSX", "true,false,,XLSX", "true,true,Env1,XLSX", "true,false,Env1,XLSX", - "false,true,,XLSX", "false,false,,XLSX", "false,true,Env1,XLSX", "false,false,Env1,XLSX",}) - @SneakyThrows - void downloadDatasets(boolean includeTimestamps, boolean hasEmptyObs, String requestedEnv, String extension) { - // Download test experiment - String envParam = "all=true"; - if (requestedEnv != null) { - envParam = "environments=" + requestedEnv; - } - Flowable> call = client.exchange( - GET(String.format("/programs/%s/experiments/%s/export?includeTimestamps=%s&%s&fileExtension=%s", - program.getId().toString(), experimentId, includeTimestamps, envParam, extension)) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class - ); - HttpResponse response = call.blockingFirst(); - - // Assert 200 response - assertEquals(HttpStatus.OK, response.getStatus()); - - // Assert file format fidelity - Map mediaTypeByExtension = new HashMap<>(); - mediaTypeByExtension.put("CSV", FileType.CSV.getMimeType()); - mediaTypeByExtension.put("XLS", FileType.XLS.getMimeType()); - mediaTypeByExtension.put("XLSX", FileType.XLSX.getMimeType()); - String downloadMediaType = response.getHeaders().getContentType().orElseThrow(Exception::new); - assertEquals(mediaTypeByExtension.get(extension), downloadMediaType); - - // Assert import/export fidelity and presence of observation units in export - ByteArrayInputStream bodyStream = new ByteArrayInputStream(response.body()); - Table download = Table.create(); - if (extension.equals("CSV")) { - download = FileUtil.parseTableFromCsv(bodyStream); - } - if (extension.equals("XLS") || extension.equals("XLSX")) { - download = FileUtil.parseTableFromExcel(bodyStream, 0); - } - checkDownloadTable(requestedEnv, rows, download, includeTimestamps, extension); - } } From 43c09d4880e07d7b287a12c51cec79b8289f827f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 6 Jun 2023 11:16:21 -0400 Subject: [PATCH 20/27] clean up TrialService and factor out duplicated code --- .../brapi/v2/services/BrAPITrialService.java | 214 +++--------------- 1 file changed, 29 insertions(+), 185 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 0f5866a3e..057f49595 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,10 +1,11 @@ package org.breedinginsight.brapi.v2.services; import io.micronaut.http.server.exceptions.InternalServerException; +import io.micronaut.context.annotation.Property; import io.micronaut.http.server.types.files.StreamedFile; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; -import lombok.extern.slf4j.Slf4j; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIListTypes; @@ -13,6 +14,7 @@ import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.Utilities; import org.brapi.v2.model.core.*; @@ -23,8 +25,6 @@ import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; import org.breedinginsight.brapps.importer.daos.*; -import org.breedinginsight.brapps.importer.model.base.AdditionalInfo; -import org.breedinginsight.brapps.importer.model.base.ObservationVariable; import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; @@ -32,19 +32,13 @@ import org.breedinginsight.model.Column; import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.Program; -import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.parsers.ParsingException; import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; import org.breedinginsight.services.writers.CSVWriter; import org.breedinginsight.services.writers.ExcelWriter; -import org.breedinginsight.utilities.FileUtil; import org.breedinginsight.utilities.Utilities; -import tech.tablesaw.api.Table; - import javax.inject.Inject; import javax.inject.Singleton; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; @@ -54,22 +48,19 @@ @Slf4j @Singleton public class BrAPITrialService { - + private final ProgramService programService; private final BrAPITrialDAO trialDAO; private final BrAPIObservationUnitDAO ouDAO; private final BrAPIObservationDAO observationDAO; - private final ProgramService programService; private final BrAPIListDAO listDAO; private final BrAPIObservationVariableDAO obsVarDAO; private final BrAPIStudyDAO studyDAO; private final BrAPISeasonDAO seasonDAO; - private final BrAPIObservationUnitDAO ouDAO; private final BrAPIGermplasmDAO germplasmDAO; @Inject public BrAPITrialService(ProgramService programService, BrAPITrialDAO trialDAO, - BrAPIObservationUnitDAO ouDAO, BrAPIObservationDAO observationDAO, BrAPIListDAO listDAO, BrAPIObservationVariableDAO obsVarDAO, @@ -77,9 +68,9 @@ public BrAPITrialService(ProgramService programService, BrAPISeasonDAO seasonDAO, BrAPIObservationUnitDAO ouDAO, BrAPIGermplasmDAO germplasmDAO) { + this.programService = programService; this.trialDAO = trialDAO; - this.ouDAO = ouDAO; this.observationDAO = observationDAO; this.listDAO = listDAO; this.obsVarDAO = obsVarDAO; @@ -137,7 +128,7 @@ public BrAPITrial getExperiment(Program program, UUID experimentId) throws ApiEx public DownloadFile exportObservations( Program program, UUID experimentId, - ExperimentExportQuery params) throws IOException, DoesNotExistException, ApiException, ParsingException { + ExperimentExportQuery params) throws IOException, DoesNotExistException, ApiException { StreamedFile downloadFile; boolean isDataset = false; List dataset = new ArrayList<>(); @@ -151,15 +142,13 @@ public DownloadFile exportObservations( // get requested environments for the experiment List expStudies = studyDAO.getStudiesByExperimentID(experimentId, program); if (!requestedEnvNames.isEmpty()) { - expStudies = expStudies.stream().filter(study -> { - return requestedEnvNames.contains(study.getStudyName()); - }).collect(Collectors.toList()); + expStudies = expStudies + .stream() + .filter(study -> requestedEnvNames.contains( + Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey()))) + .collect(Collectors.toList()); } - expStudies.stream().forEach(study -> { - if (studyByDbId.get(study.getStudyDbId()) == null) { - studyByDbId.put(study.getStudyDbId(), study); - } - }); + expStudies.forEach(study -> studyByDbId.putIfAbsent(study.getStudyDbId(), study)); // get the OUs for the requested environments List ous = new ArrayList<>(); @@ -168,7 +157,8 @@ public DownloadFile exportObservations( ous.addAll(ouDAO.getObservationUnitsForStudyDbId(study.getStudyDbId(), program)); } } catch (ApiException err) { - + log.error("Error fetching observation units for a study by its DbId" + + Utilities.generateApiExceptionLogMessage(err), err); } // make export columns including columns for requested dataset obsvars and timestamps if requested @@ -192,7 +182,7 @@ public DownloadFile exportObservations( if (!requestedEnvNames.isEmpty()) { dataset = filterDatasetByEnvironment(dataset, requestedEnvNames); } - addBrAPIObsToRecords(dataset, experiment, program, rowByOUId, params.isIncludeTimestamps(), obsVars); + addBrAPIObsToRecords(dataset, experiment, program, ous, studyByDbId, rowByOUId, params.isIncludeTimestamps(), obsVars); // make export rows for OUs without observations if (rowByOUId.size() < ous.size()) { @@ -205,10 +195,7 @@ public DownloadFile exportObservations( } // write export data to requested file format - List> exportRows = rowByOUId - .entrySet() - .stream() - .map(entry -> entry.getValue()).collect(Collectors.toList()); + List> exportRows = new ArrayList<>(rowByOUId.values()); if (fileType.equals(FileType.CSV)){ downloadFile = CSVWriter.writeToDownload(columns, exportRows, fileType); } else { @@ -220,37 +207,20 @@ public DownloadFile exportObservations( return new DownloadFile(fileName, downloadFile); } - private Map> addBrAPIObsToRecords( + private void addBrAPIObsToRecords( List dataset, BrAPITrial experiment, Program program, + List ous, + Map studyByDbId, Map> rowByOUId, boolean includeTimestamp, List obsVars) throws ApiException, DoesNotExistException { - // lookup table for export rows by OU id - // Map> rowByOUId = new HashMap<>(); - - // cache studies belonging to dataset - Map studyByDbId = new HashMap<>(); - for (BrAPIObservation obs: dataset) { - if (studyByDbId.get(obs.getStudyDbId()) == null) { - BrAPIStudy study = studyDAO.getStudyByDbId(obs.getStudyDbId(), program) - .orElseThrow(() -> new DoesNotExistException("Study not returned from BrAPI Service")); - studyByDbId.put(obs.getStudyDbId(), study); - } - } - for (BrAPIObservation obs: dataset) { - - // get ouId - List ous = ouDAO.getObservationUnitByName(List.of(obs.getObservationUnitName()), program); - if (ous.isEmpty()) { - throw new DoesNotExistException("Observation unit not returned from BrAPI service"); - } BrAPIObservationUnit ou = ous.stream() .filter(unit -> obs.getObservationUnitDbId().equals(unit.getObservationUnitDbId())) .findAny() - .orElseThrow(() -> new RuntimeException()); + .orElseThrow(RuntimeException::new); String ouId = getOUId(ou); // if there is a row with that ouId then just add the obs var data and timestamp to the row @@ -258,67 +228,12 @@ private Map> addBrAPIObsToRecords( addObsVarDataToRow(rowByOUId.get(ouId), obs, includeTimestamp, obsVars, program); } else { - // otherwise, make a new row - HashMap row = new HashMap<>(); - BrAPIGermplasm germplasm = germplasmDAO.getGermplasmByDBID(obs.getGermplasmDbId(), program.getId()) - .orElseThrow(() -> new DoesNotExistException("Germplasm not returned from BrAPI service")); - BrAPIStudy study = studyByDbId.get(obs.getStudyDbId()); - row.put(ExperimentObservation.Columns.GERMPLASM_NAME, Utilities.removeProgramKey(obs.getGermplasmName(), program.getKey(), germplasm.getAccessionNumber())); - row.put(ExperimentObservation.Columns.GERMPLASM_GID, germplasm.getAccessionNumber()); - - // use only the capitalized first character of the entry type for test/check - BrAPIEntryTypeEnum entryType = ou.getObservationUnitPosition().getEntryType(); - String testCheck = entryType != null ? String.valueOf(Character.toUpperCase(entryType.toString().charAt(0))) : null; - row.put(ExperimentObservation.Columns.TEST_CHECK, testCheck); - row.put(ExperimentObservation.Columns.EXP_TITLE, Utilities.removeProgramKey(experiment.getTrialName(), program.getKey())); - row.put(ExperimentObservation.Columns.EXP_DESCRIPTION, experiment.getTrialDescription()); - row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); - row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); - String keyedEnvName = obs.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.STUDY_NAME).getAsString(); - String envName = Utilities.removeProgramKeyAndUnknownAdditionalData(keyedEnvName, program.getKey()); - row.put(ExperimentObservation.Columns.ENV, envName); - row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); - row.put(ExperimentObservation.Columns.ENV_YEAR, obs.getSeason().getYear()); - row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getObservationUnitName(), program.getKey())); - - // get replicate number - Optional repLevel = ou.getObservationUnitPosition() - .getObservationLevelRelationships().stream() - .filter(level -> BrAPIConstants.REPLICATE.getValue().equals(level.getLevelName())) - .findFirst(); - if (repLevel.isPresent()) { - row.put(ExperimentObservation.Columns.REP_NUM, Integer.parseInt(repLevel.get().getLevelCode())); - } - - //get block number - Optional blockLevel = ou.getObservationUnitPosition() - .getObservationLevelRelationships().stream() - .filter(level -> BrAPIConstants.BLOCK.getValue().equals(level.getLevelName())) - .findFirst(); - if (blockLevel.isPresent()) { - row.put(ExperimentObservation.Columns.BLOCK_NUM, Integer.parseInt(blockLevel.get().getLevelCode())); - } - - if (ou.getObservationUnitPosition() != null) { - row.put(ExperimentObservation.Columns.ROW, Double.parseDouble(ou.getObservationUnitPosition().getPositionCoordinateX())); - row.put(ExperimentObservation.Columns.COLUMN, Double.parseDouble(ou.getObservationUnitPosition().getPositionCoordinateY())); - } - - if (ou.getTreatments() != null && !ou.getTreatments().isEmpty()) { - row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, ou.getTreatments().get(0).getFactor().toString()); - } else { - row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, null); - } - - row.put(ExperimentObservation.Columns.OBS_UNIT_ID, ouId); + // otherwise make a new row + Map row = createExportRow(experiment, program, ou, studyByDbId); addObsVarDataToRow(row, obs, includeTimestamp, obsVars, program); rowByOUId.put(ouId, row); } } - - // return a list of export rows - return rowByOUId; - //return rowByOUId.entrySet().stream().map(entry -> entry.getValue()).collect(Collectors.toList()); } private String getOUId(BrAPIObservationUnit ou) { @@ -367,8 +282,7 @@ public List getDatasetObsVars(String datasetId, Progra String listDbId = lists.get(0).getListDbId(); BrAPIListsSingleResponse list = listDAO.getListById(listDbId, program.getId()); List obsVarNames = list.getResult().getData(); - List obsVars = obsVarDAO.getVariableByName(obsVarNames, program.getId()); - return obsVars; + return obsVarDAO.getVariableByName(obsVarNames, program.getId()); } public BrAPITrial getExperiment(Program program, UUID experimentId) throws ApiException { @@ -416,25 +330,23 @@ private Map createExportRow( .getObservationLevelRelationships().stream() .filter(level -> BrAPIConstants.REPLICATE.getValue().equals(level.getLevelName())) .findFirst(); - if (repLevel.isPresent()) { - row.put(ExperimentObservation.Columns.REP_NUM, Integer.parseInt(repLevel.get().getLevelCode())); - } + repLevel.ifPresent(brAPIObservationUnitLevelRelationship -> + row.put(ExperimentObservation.Columns.REP_NUM, Integer.parseInt(brAPIObservationUnitLevelRelationship.getLevelCode()))); //get block number Optional blockLevel = ou.getObservationUnitPosition() .getObservationLevelRelationships().stream() .filter(level -> BrAPIConstants.BLOCK.getValue().equals(level.getLevelName())) .findFirst(); - if (blockLevel.isPresent()) { - row.put(ExperimentObservation.Columns.BLOCK_NUM, Integer.parseInt(blockLevel.get().getLevelCode())); - } + blockLevel.ifPresent(brAPIObservationUnitLevelRelationship -> + row.put(ExperimentObservation.Columns.BLOCK_NUM, Integer.parseInt(brAPIObservationUnitLevelRelationship.getLevelCode()))); if (ou.getObservationUnitPosition() != null && ou.getObservationUnitPosition().getPositionCoordinateX() != null && ou.getObservationUnitPosition().getPositionCoordinateY() != null) { row.put(ExperimentObservation.Columns.ROW, Double.parseDouble(ou.getObservationUnitPosition().getPositionCoordinateX())); row.put(ExperimentObservation.Columns.COLUMN, Double.parseDouble(ou.getObservationUnitPosition().getPositionCoordinateY())); } if (ou.getTreatments() != null && !ou.getTreatments().isEmpty()) { - row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, ou.getTreatments().get(0).getFactor().toString()); + row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, ou.getTreatments().get(0).getFactor()); } else { row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, null); } @@ -445,7 +357,7 @@ private Map createExportRow( - private List addObsVarColumns( + private void addObsVarColumns( List columns, List obsVars, boolean includeTimestamps, @@ -467,7 +379,6 @@ private List addObsVarColumns( columns.add(new Column(String.format("TS:%s",varName),Column.ColumnDataType.STRING)); } } - return columns; } private String makeFileName(BrAPITrial experiment, Program program, String envName) { // _Observation Dataset [-]__ @@ -486,71 +397,4 @@ private List filterDatasetByEnvironment(List .get(BrAPIAdditionalInfoFields.STUDY_NAME).getAsString())).collect(Collectors.toList()); } - private Map makeExpObsMapByHeader(ExperimentObservation obs) { - Map row = new HashMap<>(); - if (obs != null) { - row.put(ExperimentObservation.Columns.GERMPLASM_NAME, obs.getGermplasmName()); - row.put(ExperimentObservation.Columns.GERMPLASM_GID, obs.getGid()); - row.put(ExperimentObservation.Columns.TEST_CHECK, obs.getTestOrCheck()); - row.put(ExperimentObservation.Columns.EXP_TITLE, obs.getExpTitle()); - row.put(ExperimentObservation.Columns.EXP_UNIT, obs.getExpUnit()); - row.put(ExperimentObservation.Columns.EXP_TYPE, obs.getExpType()); - row.put(ExperimentObservation.Columns.ENV, obs.getEnv()); - row.put(ExperimentObservation.Columns.ENV_LOCATION, obs.getEnvLocation()); - row.put(ExperimentObservation.Columns.ENV_YEAR, obs.getEnvYear()); - row.put(ExperimentObservation.Columns.EXP_UNIT_ID, obs.getExpUnitId()); - row.put(ExperimentObservation.Columns.REP_NUM, obs.getExpReplicateNo()); - row.put(ExperimentObservation.Columns.BLOCK_NUM, obs.getExpBlockNo()); - row.put(ExperimentObservation.Columns.ROW, obs.getRow()); - row.put(ExperimentObservation.Columns.COLUMN, obs.getColumn()); - } - - return row; - } - public List getObservationDataset(Program program, UUID experimentId) throws DoesNotExistException { - List dataset = new ArrayList<>(); - try { - List trials = trialDAO.getTrialsByExperimentIds(List.of(experimentId), program); - if (trials.isEmpty()) { - throw new DoesNotExistException("Experiment not found"); - } - BrAPITrial datasetTrial = trials.stream().filter(trial -> { - return trial - .getAdditionalInfo() - .getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null; - }).findFirst().orElseThrow(() -> new DoesNotExistException("Experiment dataset not found")); - dataset = observationDAO.getObservationsByTrialDbId(List.of(datasetTrial.getTrialDbId()), program); - } catch (ApiException e) { - log.error("Error fetching BrAPI observations: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } catch (DoesNotExistException e) { - log.error("Trial does not exist", e); - throw new DoesNotExistException(e.getMessage()); - } - return dataset; - } - public List getDataset(Program program, UUID experimentId, UUID datasetId) throws ApiException, DoesNotExistException { - List dataset = new ArrayList<>(); - try { - List trials = trialDAO.getTrialsByExperimentIds(List.of(experimentId), program); - if (trials.isEmpty()) { - throw new DoesNotExistException("Experiment not found"); - } - BrAPITrial datasetTrial = trials.stream().filter(trial -> { - String id = trial - .getAdditionalInfo() - .getAsJsonObject(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); - return id != null; - }).findAny().orElseThrow(() -> new DoesNotExistException("Experiment dataset not found")); - dataset = observationDAO.getObservationsByTrialDbId(List.of(datasetTrial.getTrialDbId()), program); - } catch (ApiException e) { - log.error("Error fetching BrAPI observations: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } catch (DoesNotExistException e) { - log.error("Trial does not exist", e); - throw new DoesNotExistException(e.getMessage()); - } - return dataset; - } } From 817184eadbd1b0e139dcabacee862ad3fc88ac3e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 6 Jun 2023 11:36:45 -0400 Subject: [PATCH 21/27] clean up ExperimentController --- .../java/org/breedinginsight/brapi/v2/ExperimentController.java | 1 - .../org/breedinginsight/brapi/v2/services/BrAPITrialService.java | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index b679c653f..0dd525615 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -111,7 +111,6 @@ public HttpResponse datasetExport( try { Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program does not exist")); DownloadFile datasetFile = experimentService.exportObservations(program, experimentId, queryParams); - //Table table = FileUtil.parseTableFromCsv(datasetFile.getStreamedFile().getInputStream()); HttpResponse response = HttpResponse .ok(datasetFile.getStreamedFile()) .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + datasetFile.getFileName()); diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 057f49595..f82b92b21 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -159,6 +159,7 @@ public DownloadFile exportObservations( } catch (ApiException err) { log.error("Error fetching observation units for a study by its DbId" + Utilities.generateApiExceptionLogMessage(err), err); + err.printStackTrace(); } // make export columns including columns for requested dataset obsvars and timestamps if requested From d0384fccbcde6200b1bf1c9781749b3c16d4d3c4 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 6 Jun 2023 13:03:06 -0400 Subject: [PATCH 22/27] fix merge conflicts --- .../brapi/v2/ExperimentController.java | 14 +------ .../brapi/v2/services/BrAPITrialService.java | 41 +++---------------- 2 files changed, 6 insertions(+), 49 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index 0dd525615..826061c88 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -11,8 +11,6 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.pheno.BrAPIObservation; -import org.brapi.v2.model.pheno.BrAPIObservationVariable; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.api.model.v1.request.query.SearchRequest; @@ -23,22 +21,14 @@ import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery; import org.breedinginsight.brapi.v2.services.BrAPITrialService; -import org.breedinginsight.brapps.importer.model.base.ObservationVariable; -import org.breedinginsight.brapps.importer.model.exports.FileType; -import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.Program; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.response.ResponseUtils; import org.breedinginsight.utilities.response.mappers.ExperimentQueryMapper; -import org.breedinginsight.utilities.FileUtil; -import org.breedinginsight.utilities.Utilities; -import tech.tablesaw.api.Table; - import javax.inject.Inject; import javax.validation.Valid; -import java.util.HashMap; import java.util.List; import java.util.UUID; @@ -46,14 +36,12 @@ @Controller @Secured(SecurityRule.IS_AUTHENTICATED) public class ExperimentController { - private final ProgramDAO programDAO; private final BrAPITrialService experimentService; private final ExperimentQueryMapper experimentQueryMapper; private final ProgramService programService; @Inject - public ExperimentController(ProgramDAO programDAO, BrAPITrialService experimentService, ExperimentQueryMapper experimentQueryMapper, ProgramService programService) { - this.programDAO = programDAO; + public ExperimentController(BrAPITrialService experimentService, ExperimentQueryMapper experimentQueryMapper, ProgramService programService) { this.experimentService = experimentService; this.experimentQueryMapper = experimentQueryMapper; this.programService = programService; diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index f82b92b21..a15b0c919 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,22 +1,11 @@ package org.breedinginsight.brapi.v2.services; - -import io.micronaut.http.server.exceptions.InternalServerException; import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; import io.micronaut.http.server.types.files.StreamedFile; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.core.BrAPIListSummary; -import org.brapi.v2.model.core.BrAPIListTypes; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; -import org.breedinginsight.services.ProgramService; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.utilities.Utilities; import org.brapi.v2.model.core.*; import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; import org.brapi.v2.model.germ.BrAPIGermplasm; @@ -48,19 +37,19 @@ @Slf4j @Singleton public class BrAPITrialService { - private final ProgramService programService; + @Property(name = "brapi.server.reference-source") + private String referenceSource; private final BrAPITrialDAO trialDAO; - private final BrAPIObservationUnitDAO ouDAO; private final BrAPIObservationDAO observationDAO; private final BrAPIListDAO listDAO; private final BrAPIObservationVariableDAO obsVarDAO; private final BrAPIStudyDAO studyDAO; private final BrAPISeasonDAO seasonDAO; + private final BrAPIObservationUnitDAO ouDAO; private final BrAPIGermplasmDAO germplasmDAO; @Inject - public BrAPITrialService(ProgramService programService, - BrAPITrialDAO trialDAO, + public BrAPITrialService(BrAPITrialDAO trialDAO, BrAPIObservationDAO observationDAO, BrAPIListDAO listDAO, BrAPIObservationVariableDAO obsVarDAO, @@ -69,7 +58,6 @@ public BrAPITrialService(ProgramService programService, BrAPIObservationUnitDAO ouDAO, BrAPIGermplasmDAO germplasmDAO) { - this.programService = programService; this.trialDAO = trialDAO; this.observationDAO = observationDAO; this.listDAO = listDAO; @@ -104,25 +92,6 @@ public BrAPITrial getTrialDataByUUID(UUID programId, UUID trialId, boolean stats private long countGermplasm(UUID programId, String trialDbId) throws ApiException, DoesNotExistException{ List obUnits = ouDAO.getObservationUnitsForTrialDbId(programId, trialDbId); return obUnits.stream().map(BrAPIObservationUnit::getGermplasmDbId).distinct().count(); - public List getDatasetObsVars(String datasetId, Program program) throws ApiException, DoesNotExistException { - List lists = listDAO.getListByTypeAndExternalRef( - BrAPIListTypes.OBSERVATIONVARIABLES, - program.getId(), - String.format("%s/%s", referenceSource, ExternalReferenceSource.DATASET.getName()), - UUID.fromString(datasetId)); - if (lists == null || lists.isEmpty()) { - throw new DoesNotExistException("Dataset observation variables list not returned from BrAPI service"); - } - String listDbId = lists.get(0).getListDbId(); - BrAPIListsSingleResponse list = listDAO.getListById(listDbId, program.getId()); - List obsVarNames = list.getResult().getData(); - List obsVars = obsVarDAO.getVariableByName(obsVarNames, program.getId()); - return obsVars; - } - - public BrAPITrial getExperiment(Program program, UUID experimentId) throws ApiException { - List experiments = trialDAO.getTrialsByExperimentIds(List.of(experimentId), program); - return experiments.get(0); } public DownloadFile exportObservations( From 3dde1bb9d579fc7886fb7b2ec165ce603b7078b9 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 7 Jun 2023 15:03:03 -0400 Subject: [PATCH 23/27] improve code --- .../brapi/v2/services/BrAPITrialService.java | 86 ++++++++++++------- .../breedinginsight/utilities/FileUtil.java | 2 - .../ExperimentControllerIntegrationTest.java | 46 ++++++---- 3 files changed, 81 insertions(+), 53 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index a15b0c919..afe9ebb7b 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -37,8 +37,7 @@ @Slf4j @Singleton public class BrAPITrialService { - @Property(name = "brapi.server.reference-source") - private String referenceSource; + private final String referenceSource; private final BrAPITrialDAO trialDAO; private final BrAPIObservationDAO observationDAO; private final BrAPIListDAO listDAO; @@ -49,7 +48,8 @@ public class BrAPITrialService { private final BrAPIGermplasmDAO germplasmDAO; @Inject - public BrAPITrialService(BrAPITrialDAO trialDAO, + public BrAPITrialService(@Property(name = "brapi.server.reference-source") String referenceSource, + BrAPITrialDAO trialDAO, BrAPIObservationDAO observationDAO, BrAPIListDAO listDAO, BrAPIObservationVariableDAO obsVarDAO, @@ -58,6 +58,7 @@ public BrAPITrialService(BrAPITrialDAO trialDAO, BrAPIObservationUnitDAO ouDAO, BrAPIGermplasmDAO germplasmDAO) { + this.referenceSource = referenceSource; this.trialDAO = trialDAO; this.observationDAO = observationDAO; this.listDAO = listDAO; @@ -104,31 +105,31 @@ public DownloadFile exportObservations( List obsVars = new ArrayList<>(); Map> rowByOUId = new HashMap<>(); Map studyByDbId = new HashMap<>(); - List requestedEnvNames = StringUtils.isNotBlank(params.getEnvironments()) ? + List requestedEnvIds = StringUtils.isNotBlank(params.getEnvironments()) ? new ArrayList<>(Arrays.asList(params.getEnvironments().split(","))) : new ArrayList<>(); FileType fileType = params.getFileExtension(); // get requested environments for the experiment List expStudies = studyDAO.getStudiesByExperimentID(experimentId, program); - if (!requestedEnvNames.isEmpty()) { + if (!requestedEnvIds.isEmpty()) { expStudies = expStudies .stream() - .filter(study -> requestedEnvNames.contains( - Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey()))) + .filter(study -> requestedEnvIds.contains(getStudyId(study))) .collect(Collectors.toList()); } expStudies.forEach(study -> studyByDbId.putIfAbsent(study.getStudyDbId(), study)); // get the OUs for the requested environments List ous = new ArrayList<>(); + Map ouByOUDbId = new HashMap<>(); try { for (BrAPIStudy study: expStudies) { ous.addAll(ouDAO.getObservationUnitsForStudyDbId(study.getStudyDbId(), program)); + ous.forEach(ou -> ouByOUDbId.put(ou.getObservationUnitDbId(), ou)); } } catch (ApiException err) { log.error("Error fetching observation units for a study by its DbId" + Utilities.generateApiExceptionLogMessage(err), err); - err.printStackTrace(); } // make export columns including columns for requested dataset obsvars and timestamps if requested @@ -149,10 +150,19 @@ public DownloadFile exportObservations( if (isDataset) { dataset = observationDAO.getObservationsByTrialDbId(List.of(experiment.getTrialDbId()), program); } - if (!requestedEnvNames.isEmpty()) { - dataset = filterDatasetByEnvironment(dataset, requestedEnvNames); + if (!requestedEnvIds.isEmpty()) { + dataset = filterDatasetByEnvironment(dataset, requestedEnvIds, studyByDbId); } - addBrAPIObsToRecords(dataset, experiment, program, ous, studyByDbId, rowByOUId, params.isIncludeTimestamps(), obsVars); + addBrAPIObsToRecords( + dataset, + experiment, + program, + ouByOUDbId, + studyByDbId, + rowByOUId, + params.isIncludeTimestamps(), + obsVars + ); // make export rows for OUs without observations if (rowByOUId.size() < ous.size()) { @@ -169,7 +179,7 @@ public DownloadFile exportObservations( if (fileType.equals(FileType.CSV)){ downloadFile = CSVWriter.writeToDownload(columns, exportRows, fileType); } else { - downloadFile = ExcelWriter.writeToDownload("Dataset Export", columns, exportRows, fileType); + downloadFile = ExcelWriter.writeToDownload("Experiment Data", columns, exportRows, fileType); } String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); @@ -181,26 +191,30 @@ private void addBrAPIObsToRecords( List dataset, BrAPITrial experiment, Program program, - List ous, + Map ouByOUDbId, Map studyByDbId, Map> rowByOUId, boolean includeTimestamp, List obsVars) throws ApiException, DoesNotExistException { + Map varByDbId = new HashMap<>(); + obsVars.forEach(var -> varByDbId.put(var.getObservationVariableDbId(), var)); for (BrAPIObservation obs: dataset) { - BrAPIObservationUnit ou = ous.stream() - .filter(unit -> obs.getObservationUnitDbId().equals(unit.getObservationUnitDbId())) - .findAny() - .orElseThrow(RuntimeException::new); + + // get observation unit for observation + BrAPIObservationUnit ou = ouByOUDbId.get(obs.getObservationUnitDbId()); String ouId = getOUId(ou); + // get observation variable for BrAPI observation + BrAPIObservationVariable var = varByDbId.get(obs.getObservationVariableDbId()); + // if there is a row with that ouId then just add the obs var data and timestamp to the row if (rowByOUId.get(ouId) != null) { - addObsVarDataToRow(rowByOUId.get(ouId), obs, includeTimestamp, obsVars, program); + addObsVarDataToRow(rowByOUId.get(ouId), obs, includeTimestamp, var, program); } else { // otherwise make a new row Map row = createExportRow(experiment, program, ou, studyByDbId); - addObsVarDataToRow(row, obs, includeTimestamp, obsVars, program); + addObsVarDataToRow(row, obs, includeTimestamp, var, program); rowByOUId.put(ouId, row); } } @@ -214,17 +228,20 @@ private String getOUId(BrAPIObservationUnit ou) { return ouXref.getReferenceID(); } + private String getStudyId(BrAPIStudy study) { + BrAPIExternalReference studyXref = Utilities.getExternalReference( + study.getExternalReferences(), + String.format("%s/%s", referenceSource, ExternalReferenceSource.STUDIES.getName())) + .orElseThrow(() -> new RuntimeException("study id not found")); + return studyXref.getReferenceID(); + } + private void addObsVarDataToRow( Map row, BrAPIObservation obs, boolean includeTimestamp, - List obsVars, + BrAPIObservationVariable var, Program program) { - // get observation variable for BrAPI observation - BrAPIObservationVariable var = obsVars.stream() - .filter(obsVar -> obs.getObservationVariableName().equals(obsVar.getObservationVariableName())) - .collect(Collectors.toList()).get(0); - String varName = Utilities.removeProgramKey(obs.getObservationVariableName(), program.getKey()); if (var.getScale().getDataType().equals(BrAPITraitDataType.ORDINAL)) { row.put(varName, Integer.parseInt(obs.getValue())); @@ -257,6 +274,10 @@ public List getDatasetObsVars(String datasetId, Progra public BrAPITrial getExperiment(Program program, UUID experimentId) throws ApiException { List experiments = trialDAO.getTrialsByExperimentIds(List.of(experimentId), program); + if (experiments.isEmpty()) { + throw new RuntimeException("A trial with given experiment id was not returned"); + } + return experiments.get(0); } @@ -335,9 +356,6 @@ private void addObsVarColumns( for (BrAPIObservationVariable var: obsVars) { Column obsVarColumn = new Column(); obsVarColumn.setDataType(Column.ColumnDataType.STRING); - if (var.getScale().getDataType().equals(BrAPITraitDataType.ORDINAL)) { - obsVarColumn.setDataType(Column.ColumnDataType.INTEGER); - } if (var.getScale().getDataType().equals(BrAPITraitDataType.NUMERICAL) || var.getScale().getDataType().equals(BrAPITraitDataType.DURATION)) { obsVarColumn.setDataType(Column.ColumnDataType.DOUBLE); @@ -361,10 +379,14 @@ private String makeFileName(BrAPITrial experiment, Program program, String envNa envName, timestamp); } - private List filterDatasetByEnvironment(List dataset, List envNames) { - return dataset.stream().filter(obs -> envNames.contains( - obs.getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.STUDY_NAME).getAsString())).collect(Collectors.toList()); + private List filterDatasetByEnvironment( + List dataset, + List envIds, + Map studyByDbId) { + return dataset + .stream() + .filter(obs -> envIds.contains(getStudyId(studyByDbId.get(obs.getStudyDbId())))) + .collect(Collectors.toList()); } } diff --git a/src/main/java/org/breedinginsight/utilities/FileUtil.java b/src/main/java/org/breedinginsight/utilities/FileUtil.java index 085115973..37f401f76 100644 --- a/src/main/java/org/breedinginsight/utilities/FileUtil.java +++ b/src/main/java/org/breedinginsight/utilities/FileUtil.java @@ -40,7 +40,6 @@ public class FileUtil { // For backward compatibility private static final String OLD_GERMPLASM_EXCEL_DATA_SHEET_NAME = "Germplasm Import"; private static final String OLD_EXPERIMENT_EXCEL_DATA_SHEET_NAME = "Experiment Data"; - private static final String DATASET_EXPORT_EXCEL_DATA_SHEET_NAME = "Dataset Export"; public static Table parseTableFromExcel(InputStream inputStream, Integer headerRowIndex) throws ParsingException { @@ -57,7 +56,6 @@ public static Table parseTableFromExcel(InputStream inputStream, Integer headerR //For backward compatibility allow old sheet names if( sheet == null){ sheet = workbook.getSheet(OLD_GERMPLASM_EXCEL_DATA_SHEET_NAME); } if( sheet == null){ sheet = workbook.getSheet(OLD_EXPERIMENT_EXCEL_DATA_SHEET_NAME); } - if( sheet == null){ sheet = workbook.getSheet(DATASET_EXPORT_EXCEL_DATA_SHEET_NAME); } if (sheet == null) { throw new ParsingException(ParsingExceptionType.MISSING_SHEET); diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index af335d294..6545e92ee 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -56,6 +56,7 @@ public class ExperimentControllerIntegrationTest extends BrAPITest { private Program program; private String experimentId; + private List envIds = new ArrayList<>(); private final List> rows = new ArrayList<>(); private final List columns = ExperimentFileColumns.getOrderedColumns(); private List traits; @@ -179,6 +180,16 @@ void setup() throws Exception { .get(0).getAsJsonObject() .get("trial").getAsJsonObject() .get("id").getAsString(); + envIds.add(importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("study").getAsJsonObject() + .get("brAPIObject").getAsJsonObject() + .get("externalReferences").getAsJsonArray() + .get(2).getAsJsonObject() + .get("referenceID").getAsString() + ); } /* @@ -197,18 +208,18 @@ void setup() throws Exception { - export populated dataset, multiple environment, xlsx format */ @ParameterizedTest - @CsvSource(value = {"true,,CSV", "true,Env1,CSV", - "false,,CSV", "false,Env1,CSV", - "true,,XLS", "true,Env1,XLS", - "false,,XLS", "false,Env1,XLS", - "true,,XLSX", "true,Env1,XLSX", - "false,,XLSX", "false,Env1,XLSX",}) + @CsvSource(value = {"true,false,CSV", "true,true,CSV", + "false,false,CSV", "false,true,CSV", + "true,false,XLS", "true,true,XLS", + "false,false,XLS", "false,true,XLS", + "true,false,XLSX", "true,true,XLSX", + "false,false,XLSX", "false,true,XLSX",}) @SneakyThrows - void downloadDatasets(boolean includeTimestamps, String requestedEnv, String extension) { + void downloadDatasets(boolean includeTimestamps, boolean requestEnv, String extension) { // Download test experiment String envParam = "all=true"; - if (requestedEnv != null) { - envParam = "environments=" + requestedEnv; + if (requestEnv) { + envParam = "environments=" + String.join(",", envIds); } Flowable> call = client.exchange( GET(String.format("/programs/%s/experiments/%s/export?includeTimestamps=%s&%s&fileExtension=%s", @@ -237,7 +248,7 @@ void downloadDatasets(boolean includeTimestamps, String requestedEnv, String ext if (extension.equals("XLS") || extension.equals("XLSX")) { download = FileUtil.parseTableFromExcel(bodyStream, 0); } - checkDownloadTable(requestedEnv, rows, download, includeTimestamps, extension); + checkDownloadTable(requestEnv, rows, download, includeTimestamps, extension); } private File writeDataToFile(List> data, List traits) throws IOException { File file = File.createTempFile("test", ".csv"); @@ -325,13 +336,13 @@ private List createGermplasm(int numToCreate) { return germplasm; } private void checkDownloadTable( - String requestedEnv, + boolean requestEnv, List> importRows, Table table, boolean includeTimestamps, String extension) { // Filename is correct: _Observation Dataset [-]__ - List> requestedImportRows; + List> requestedImportRows = importRows; // All columns included Integer expectedColNumber = columns.size(); @@ -340,8 +351,7 @@ private void checkDownloadTable( } assertEquals(expectedColNumber, table.columnCount()); - if (requestedEnv == null) { - requestedImportRows = importRows; + if (!requestEnv) { // All environments downloaded importRows.stream() @@ -355,11 +365,9 @@ private void checkDownloadTable( // Only requested environment downloaded requestedImportRows = importRows .stream() - .filter(row -> row.get("Env").toString().equals(requestedEnv)).collect(Collectors.toList()); - + .filter(row -> row.get("Env").toString().equals("Env1")).collect(Collectors.toList()); assertEquals(1, table.stringColumn("Env").countUnique()); - assertTrue(table.stringColumn("Env").contains(requestedEnv)); - + assertTrue(table.stringColumn("Env").contains("Env1")); } List> matchingImportRows = new ArrayList<>(); @@ -380,7 +388,7 @@ private void checkDownloadTable( return row.get(gid).equals(downloadRow.getString(gid)) && row.get(env).equals(downloadRow.getString(env)); } }).findAny(); - assertTrue(matchingImportRow.isPresent()); + assertTrue(matchingImportRow.isPresent() && !matchingImportRow.get().isEmpty()); // then check the rest of the fields match if (isMatchedRow(matchingImportRow.get(), downloadRow)) { From 362e065406e9f41b03077016bc6ac363c50df304 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 7 Jun 2023 18:11:07 -0400 Subject: [PATCH 24/27] improve code and fix bugs --- .../brapi/v2/services/BrAPITrialService.java | 12 ++++-- .../experiment/ExperimentFileColumns.java | 4 +- .../utilities/response/ResponseUtils.java | 10 ----- .../ExperimentControllerIntegrationTest.java | 10 +++-- .../feature1465EmptyDatasetImport.csv | 3 -- .../feature1465EmptyDatasetImport.xls | Bin 36352 -> 0 bytes .../feature1465EmptyDatasetImport.xlsx | Bin 12775 -> 0 bytes .../feature1465PopulatedDatasetImport.csv | 3 -- .../feature1465PopulatedDatasetImport.xls | Bin 36352 -> 0 bytes .../feature1465PopulatedDatasetImport.xlsx | Bin 12786 -> 0 bytes .../germplasm_import.csv | 2 - .../ExperimentControllerIntegrationTest.sql | 37 ------------------ 12 files changed, 16 insertions(+), 65 deletions(-) delete mode 100644 src/test/resources/files/experiment_controller/feature1465EmptyDatasetImport.csv delete mode 100644 src/test/resources/files/experiment_controller/feature1465EmptyDatasetImport.xls delete mode 100644 src/test/resources/files/experiment_controller/feature1465EmptyDatasetImport.xlsx delete mode 100644 src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.csv delete mode 100644 src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.xls delete mode 100644 src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.xlsx delete mode 100644 src/test/resources/files/experiment_controller/germplasm_import.csv delete mode 100644 src/test/resources/sql/ExperimentControllerIntegrationTest.sql diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index afe9ebb7b..c95cac171 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -132,8 +132,10 @@ public DownloadFile exportObservations( Utilities.generateApiExceptionLogMessage(err), err); } - // make export columns including columns for requested dataset obsvars and timestamps if requested + // make columns present in all exports List columns = ExperimentFileColumns.getOrderedColumns(); + + // add columns for requested dataset obsvars and timestamps BrAPITrial experiment = getExperiment(program, experimentId); if ((StringUtils.isBlank(params.getDataset()) || "observations".equalsIgnoreCase(params.getDataset())) && experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null) { @@ -142,6 +144,8 @@ public DownloadFile exportObservations( .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); isDataset = true; obsVars = getDatasetObsVars(obsDatasetId, program); + + // make additional columns in the export for each obs variable and obs variable timestamp addObsVarColumns(columns, obsVars, params.isIncludeTimestamps(), program); } @@ -253,7 +257,7 @@ private void addObsVarDataToRow( } if (includeTimestamp) { - row.put(String.format("TS:%s",varName), obs.getObservationTimeStamp()); + row.put(String.format("TS:%s",varName), obs.getObservationTimeStamp().toString()); } } @@ -333,8 +337,8 @@ private Map createExportRow( row.put(ExperimentObservation.Columns.BLOCK_NUM, Integer.parseInt(brAPIObservationUnitLevelRelationship.getLevelCode()))); if (ou.getObservationUnitPosition() != null && ou.getObservationUnitPosition().getPositionCoordinateX() != null && ou.getObservationUnitPosition().getPositionCoordinateY() != null) { - row.put(ExperimentObservation.Columns.ROW, Double.parseDouble(ou.getObservationUnitPosition().getPositionCoordinateX())); - row.put(ExperimentObservation.Columns.COLUMN, Double.parseDouble(ou.getObservationUnitPosition().getPositionCoordinateY())); + row.put(ExperimentObservation.Columns.ROW, ou.getObservationUnitPosition().getPositionCoordinateX()); + row.put(ExperimentObservation.Columns.COLUMN, ou.getObservationUnitPosition().getPositionCoordinateY()); } if (ou.getTreatments() != null && !ou.getTreatments().isEmpty()) { row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, ou.getTreatments().get(0).getFactor()); diff --git a/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java index d12da2399..5ddb65a76 100644 --- a/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java +++ b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java @@ -39,8 +39,8 @@ public enum ExperimentFileColumns { EXP_UNIT_ID(ExperimentObservation.Columns.EXP_UNIT_ID, Column.ColumnDataType.STRING), REP_NUM(ExperimentObservation.Columns.REP_NUM, Column.ColumnDataType.INTEGER), BLOCK_NUM(ExperimentObservation.Columns.BLOCK_NUM, Column.ColumnDataType.INTEGER), - ROW(ExperimentObservation.Columns.ROW, Column.ColumnDataType.DOUBLE), - COLUMN(ExperimentObservation.Columns.COLUMN, Column.ColumnDataType.DOUBLE), + ROW(ExperimentObservation.Columns.ROW, Column.ColumnDataType.STRING), + COLUMN(ExperimentObservation.Columns.COLUMN, Column.ColumnDataType.STRING), TREATMENT_FACTORS(ExperimentObservation.Columns.TREATMENT_FACTORS, Column.ColumnDataType.STRING), OBS_UNIT_ID(ExperimentObservation.Columns.OBS_UNIT_ID, Column.ColumnDataType.STRING); diff --git a/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java b/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java index 9798379fb..2a6194690 100644 --- a/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java +++ b/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java @@ -88,16 +88,6 @@ public static HttpResponse> getSingleResponse(Object data) { return HttpResponse.ok(new Response(metadata, data)); } - // File download - public static HttpResponse getExportResponse() { - return processExportResponse(); - } - - private static HttpResponse processExportResponse() { - HttpResponse response = HttpResponse.status(HttpStatus.OK).contentType(MediaType.forFilename("download.xlsx")); - return response; - } - private static HttpResponse>> processSearchResponse( List data, SearchRequest searchRequest, QueryParams queryParams, AbstractQueryMapper mapper, Metadata metadata) { diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index 6545e92ee..2dc8ed364 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -161,6 +161,7 @@ void setup() throws Exception { Float val1 = random.nextFloat(); row1.put(trait.getObservationVariableName(), val1); + // row1.put("TS:" + trait.getObservationVariableName(), "2023-06-07T20:47:23-0400"); } rows.add(row1); @@ -255,10 +256,11 @@ private File writeDataToFile(List> data, List traits) if(traits != null) { traits.forEach(trait -> columns.add( - Column.builder() - .value(trait.getObservationVariableName()) - .dataType(Column.ColumnDataType.STRING) - .build())); + Column.builder() + .value(trait.getObservationVariableName()) + .dataType(Column.ColumnDataType.STRING) + .build()) + ); } ByteArrayOutputStream byteArrayOutputStream = CSVWriter.writeToCSV(columns, data); FileOutputStream fos = new FileOutputStream(file); diff --git a/src/test/resources/files/experiment_controller/feature1465EmptyDatasetImport.csv b/src/test/resources/files/experiment_controller/feature1465EmptyDatasetImport.csv deleted file mode 100644 index 1ed2eb60c..000000000 --- a/src/test/resources/files/experiment_controller/feature1465EmptyDatasetImport.csv +++ /dev/null @@ -1,3 +0,0 @@ -Germplasm Name,Germplasm GID,Test (T) or Check (C ),Exp Title,Exp Description,Exp Unit,Exp Type,Env,Env Location,Env Year,Exp Unit ID,Exp Replicate #,Exp Block #,Row,Column,Treatment Factors,ObsUnitID,ObsVar1 -,1,,exp,,plant,trial,env1,here,2022,1,1,2,,,,, -,1,,exp,,plant,trial,env2,here,2022,1,1,2,,,,, \ No newline at end of file diff --git a/src/test/resources/files/experiment_controller/feature1465EmptyDatasetImport.xls b/src/test/resources/files/experiment_controller/feature1465EmptyDatasetImport.xls deleted file mode 100644 index c94875321d543122970bc8f6bfb7c2102b1dadec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36352 zcmeHQ349b)(tne45(o)bIE4lRB#;9VF1d0OjvNp!5im@W4jD*h!ptOug_tY^1w}+a zIaNTB0CEd*-=Z99Kq8AQ2m%Vrp|UEniuwN4Jw4OYGZ})ruHX0j()smFzv}m@>Q&XN zs@LyLr~K2pmsf6SdW9*DzRZ>VR_Vn&WN;O(a=`1dCIWBugwOiI=O&@%Ug5J3tH(-RkGj3BpA+2!tr*Uy zT%v?}FA1Nmh0i+pRI&GkHeU#k&a4Id4)xV+9yJ4TPgN|_upFjk>CAx7|7H!BNAR%% zELIl(`y!_w(dCgfdOxCuibXo3+-mX;iIrqjwQlc$Fpo^GTW>Q*5q7AjCot| zDeb<6TXW_P4YAe%t=j~*Y11h!sMnZ4sW2v}g)>z?h(3W811E3PfwO55$%0u=TVi4lZ0QNC$w)>vI1$m{(6+f?4QGb0E@T%}vRewtrc>ZX9b_sNm%828|qL=+uz_(Wcf4>U&wkqIaf9>gt z{CL^%vq8Y)Ex{Mot-Gk!qFT&N)JrW3JS*!!%L6T$o5+tRe2Cx&enti1knA(43V3i8 z@D2{)wSiN>|AQGl6Y={A9b=$Ef8++TwGS>YUGC5`Ri3c7Wr z`WE?!_ES%SpJn54*;K+)UxHisRM_Y8x^?|)`9sd)xN9KQ2Yyk`(zx()C?Dj)4&{Rm zitw{Y8ChS891t}f)(2ShGn|+t zZX<_4fgk(&fIGwwxI=l0cm(^gp)0R1m#3T`Ib2rX29K=11Gqh1)Q@7~f%?Mv*#%71_Y)rVH~y{@{u$n(k2Yg|wwZ9b|eg_6?}YV>J*b1dl<17&sYC!Sju^Ld|d8`H$F&?V{g_6f=Kr!aA8c^_Y z+8|mn9w;igRy`ivgn`0mk;#cO2#zaKE0IvBss<7=!>W-`T&o5WGR>-yP!OyJ5;E7S zkx*o;1`;yis*zB*tOgP?>#C7ZEO;c6Q;E*CLxH#&NStd2=}L8wIM)t>U3HK+*A5b% z>L78hoo3ZT;#@nJ_dlAG@CfJH!E=G?AaSmp7S%)ITsz3zs>6wM?QqpY;#@nes)xk6 zcJL@g)*TdVBfaR^7;~Ue-$yuNbcRcXF=j=MqFH6T*;XawF8)! z3 zr+I-gPY!3Dg0)XeI-!~v_^1J9r?w3w8$0%`+{To-4IW#{ z+z6@EhAP<(0iC-MJga>)lFCTuNFu6(L>X?BdhG%vRztAsj=ec|MFZ;?i;OIAEX2xF zOYF?~KR3pdO6g`1g~Nd_4d}`em4zvL^XP7aWckP{%%k$wav2ikXzyVW56RMQJ}KE) zW#uUg63GQ^-MZCow98@AXeStO3x_n?5S9>RmS-m^`v$#i8YoRwRiqxS2nW5TYIT2wJCnRiFW(y->OE79rhdsxUfLe#S17RyUjzWX~ zD#yx!6b%NfVbhLdFlF9_MtZkw;t)TMI_S1@=**(OgWZ{(m5qp zhk^fO>{mw86fp3+)bdA0Qd%N?q+0$+sSK%}RQ}jVs#I0#?`7(b96>|>+p1EIpkdWv z;Ex=_!>hx?@ z(g)kbIUfJ11nsXCnd`4?gS8ojZ=n)K`+}Vo#=n6EgJl z#4I#VKCANHDaT|QD9LoSCll&~3_Wu(lTq_o7jNBnOs1idOgDQn;ZDeiPf-M+dGT3a zefx!DGX6xSi*rkB2{K)(VQHrl-#hk=V=|4DWE@yp8z*GsmUb%f=aU;8lWD9Z$A9GToe88V54ns$ps25p&m8p_?a^WE@ypfD*R!t z+|t4$mL2}uF_~6MG7c=QixV<(OAC)!TRztjncCQAjfFBx+>BPl%_k~7tBjS5#Tkx@ zNGVX_FvrT;tc?{ZEJ}&Al>({UiWs|7)Y&o804q}1j}mEH1yZ@qFh*-kLR43B+SZB` z)}lljq(CaS5OnzIr5}DhScu<}k?wVZ{`FvF3S>SE z>=eQtDA$*?Ps%o%mlyhm6~P3(9){Y>7O$#lr5j6t(DVvZrNvAM7l=u?>fq-Wcn;H_ z*JNsS<2lR~6pXt1L9+e95ZS&KjE)C`;@T&n^TdFO8#X~iN2cfr&O}GJz@-i*88YPX zeM;nY9UvF|UlqtJBUkObf{qaK2sv{2Kqc~e4v>p}unOd2YTzOu?;=MIU#M^huJs)t zCx0kv%tkB=T)bIVD3T4(<>kW^@@-Y-(g6J#0LkXT7D?&W;ZKXFlG12}_1_%+en^nu zmlCzQiF~Gc{0~wv(-?k-XuKrrP5gXyx-AYDZ;ThhVFFVO{gXT|EzC+TjTTj{j#ey5E=?>+E=?>g zaH++f#8WMnl5Y|;K3PGuAyB6XGO*XV+~t#pw9z>Mtel7?i4(CTaUzz;VfHL>hwK7u zS#WoUA_k++2Ft{sL`J6Ci%fNx)|3Njb+&Eg>I?M@gLRmoIFgsWwW|n4ZvlngL>8cU z_Tbt?KtTc*weKbvmCf@7lDp(b#NJ9JGO@a>4;&90sWIrtWr^-J(w0$|TJY2(LBRk+ zmb{iDw@N`=knmjfQ*5s6A;~=St}gzD0ZF>h+o522ig1;LzLPV|huwukidt46Sdg`b z{MUxv*f`-gwKLJH!%vZvJy>ddbX-#WO#Ff{M?(-_iWs~>n~Uc)T%5+FS*mW#mde-> zEt*kmVQ=9_%5TZ;5rR3ea25Uzsnj?zE&qoL`<2!6r9nYk5ioy>)#v2r>Ns-syaDAA z8iQ7op2Ks)rsVNJm&#Ad*Bbatc!)UOm|@W7QRaf3#tFP37vap9%Oz`a`T9b1GxMs+ z;o^9cMw^3Y@5%aiU^m%3r$a4HpT(JQJ>JqbohNPa1)7|EjS222L!;x;c}}a#$jQ${ zS*sHon0TUP;)A*Le3PvCWNl6kmz*+;+T?0X8QExJ(hE&8A+L0ek>}EaKrU&3O{W>tlt?eB1OMM{x`GU;!xXuKJ=|~_l z5#o>&K;2xTJ_9aNufuYybZHqOx&p01ucILQI}INANAVg1qBwnV8G3_(H|FVef@?L0 zBmUo_g6Jy^b8bM~QB2M9F;YU{_cMiHt;xyD*685M4B8B3)Ug?Lhk zO>z&xT#j}kFZc)!h|_rkf&*d=UTwlt;F5?Lkst6Fv%f=hW?z`f=jCXj9iD4b&e%`^ zA6$EmnXIiJ1lVA%4{ajE^h227MVE#&g2&97+!_T=WD?e{i$OEO(wMgfwj7dRCZ{oS zc^ZR>$Vn8Cs2KG5d0gk<@L&o@5x=98X3fQr2jkittfGU0gEQK3O~Fp^_Mx1B0Bnk7 zGxx?kK0})|h0CQZzyu56I*Yy%`9w6rwzdC8E?ZN;a~czu!)p-bEcG5cM8kik>L+8y z++T0d{zb1tI?n}83DF>53E~i;v^319C*g5iECdSv!Iv>l0~^8QN6a_Td<1VWLG5T< zXl|UL*BdZo$PyekM})H^0a=!qDGGrshzL`LN$C(TB+Xfx4CFQTf)&j{lLyO#Obg68 zFXEnH^vEtu+8pg)kOe{OlYz_tn;1`dxG@xuE>hBs6mSQ`C9CL~Hx!7p35ek-=7g4< z?NM2wRY?$rUWQH>tD)S;Yz(+OcoaA&j6tYXOwerFN!c0=QDg_OhfL(PqzEDV6Y{z| zAkri3!bJ#quaqz&akKQuD5cMkjR6WKWl&`!59df}yvPV*Gz2Yc1%**!FXibZ0=<;4 ze8GbNGcUSEzL2a1@!d?p7>b7kgNaM7GsG-hQJN4;(}-YrEddYFG55+m zO#m&c*o>iDWM*P#92_iGpQ9f?#iDg-f)L83NVlNyTq7ss;YbTJ`7Est@-+9+oa~!( zZ<;zdo>ovUO1x$ZQ!C~4#KHz8o(xY3a&Ju;d zQ8+@9l0xF*hGl2>%+2j-G|DPm|{h>AwIFu?nX102>8FD2`EP9dbU_!ja=JmkTAhKFgvUNODx3=f8} z126(#XYnh%5{Z?XoyVSVBn<2uY~gOIVr*;_WAjk`WGr6b2In;p^ME4InlON|*HLlR zAjT>%7tT=o<3vOAfoY6zxUkpMBWu4VyW=y*g$*9$>ec|$eFC>)H!t}f!1D&y9Ft|t zqS@$Jh#x~ejdEz?*wDh8PVo7N`)~BRYS`D(bNi9S%hwFtbT6i3z^!HpV^@FKJ>~p{ z5wri=v~h3EYTv~_?B2gOu>12R17dyzz-!`^=qZXgTlgGym%0ahaif*IjpPe&VSOb1fIY(d2Ts z#Ls^8Pd(Rt+{Ty7j#q9w(&^M2DTj(T1z(SM({*2%sJ)iHvMlWC(0%jD_;qJnlyBep z;PZ#*ihV~XPI=9L20HixPWtUA#z-z7Ci_gnS^{R=WFfUrC3zp={GzTDt(-jXeD{n! zf$`~EPj1}v6N_EmH2T98XWqPU?)udk-9P!VLB+B`KBIz{d-}Y1m|4&u;JG?3mYGQ@1^=*4*1A8q^M2ByqO!eU*taD-m#+9x4{>m*%dIM&x;8o zZ+vz?=&Sf!9ou=k^l;34@|oPzpJK#pZpj#p1{vF87djK$w?_@qH|y@VztN52r*7}t zl6k9clUH1CHEh2nr%h_=wf+OnhKAfVM5h**I?f*&=I;G=e3<)RZmkS#x%!xQ-!4gWA2U1IC?th#!EMjWvv{1ewN?tqOg8-%lb5b`MBTIqSa@< z92KY^x^vFz!L?#fOdZ&DM?jZ?E(gkJ;iY$5ojYA@+_mdq_SUucuN~qpyf>`Fq7HkOJnOT<(D2;G9Ubos zd?&@VJWt&|Y1iL+u3z@?ndTEq&TOi4al?d(>$9);JRIF|`Q(%T$hguqup;g0Lz8Qi&m3CRYh9 z%fEYi*V(nYPhR~daMp9dyU+S(_@B(*;=eNEST8m>>V<&WeV!cjU1OhCFD#oiU)BCn zU#}STUbhbt4;nPv%7bza75^A?Q~$+V+0EL-KK;E*yL5iQ_un6#w;=BGhn_1cx^wz3 z_dYrGw}0J!pdTOh%}Q_W$zmW#6^W*Z=07!e8#JpS&q;-^07>Ba8oXvwZ)_(+5lM4=9~Exg=#@Lhy9g zobdtgc4p{-Endgwe%7b$(XA5~)qb*mPVl}pM<;S$bs6wx?RTS2%>1dp+p|k{si&_G zxnO#;(5K(nHM%SHn|ZzOzN*f>qQsDhOR+;|w_bFj?+azx?}`Fq8y&wi$2V!r=>FfC zN`n6RMVsK#L0`{1GR0?2WNGcI1ydJ&(z@C4wo7)J?zTCWnE7JT(pE3jYqTTbtMa3l z8r4@1x%Ncoo6{#RDSEza;&-kuoZJ12*P++;e466+a;sfS7v5t-4kYe6vgm%NZC}0L z{jH4Bv7@pUC5$Mzc>QRTCtlKyoB31UV+~$kRrlDXtH0Fo@Oka2Zl4Z_RJU4xTQhQ8 zpQwU&*WGGdD|^APv~5dU-=DoXX2CkBoZWw#?w{Mv z{OLmr$kh=!tKS{8qvx&E6PHqFY^!s>WzNK>vzxZmlm)DR^42@+cio)u zYh}^MZFis9f0O?_zj^n4bA0<8OA3p4^W8BiVGG_5D?eYRP3U?3bi|w4cRm`Q7G~q0zz8`=+ExW2g4c-`(YG z=w%E*NzB_xw>`##_OTmZv+j!@sNAZ(6r)0%k)XZ(`NkL zpj)wJNl^Cf*FS%FVnmaetXcDSM?N>Z^R$=h&3aw51)FADHm{`WdeVEStV(>XjR3o0SI@mvTo>RICcWJWE|TX>j6(CtH8l zE_hJplwU&{zBA>Eu4(Z%8xK5JHz}serogQ5lq)@79dG(!c$Vwc3D0JAD$~rL(yF=J zx`RvmZC|Gy(KWTs#PxR{#GEknJ9BE{JFgBOR$&_7dE)c&O`h6%z2b$vhbCy!KhkE+ zOc{FLaAHG<;bQ8W^M-qMD$I&ZsMr@3-Kp&QvQcLnT&?KxuHmO?6&s&;(e;hkJFVjn zfAp_cXMWLk58w6ht|x+@S$huxJA3#{;pveR2HhI@_1B|wyY+bP+Ue<`lLxLY-&lG+ zG@;z@M^noD)jlD+|A_|~4&t}Z>3MJNz@27@?(G#JP)u5cpeT#t<|HP``*5cQrT3># zR_pbj?e=@6-Njku8_v3xUhC@9Eiz!p?K?gjgVmwWYxXbE?pgHu@%kCzmpW&!YpU*f z^VLa959RK>JvS=VKY8}~UYVQ6O`ogpey8F1R|W+pwF^iM4DGzwP^bOC1QQC!4aR;pZ!oUq(tvIY2WFh?Z=)D1A4iLV z^v2)X_{_u#Bwg3VCoM9n*{^HAKa-R`I5JIucM#wmh0o~%q(qq12q;-wtTHnerTQ!G z9KwW$(fg6v454zjLEwtYgug>QYuN>N)-|H{En@yz_=w~gvVP8mvcVEkW$Xvy0kzc@pO(}Fx??r8>U1-6UuCEb7yt_G)5sWnG&2C`A zX@yY*Z}zsJ{}OXOs^HDagtkoOE3_q9d$VDL0<=XKX25nZLZobqsj(f51nA9#cLhP> zB0gBG9X_%yB|e&#d9z=Ien`B~v|0(nf~y^jWaZ6%Hp^v%&^MB!H{;B55n&`jH-!xF zWB{;6)Ex`Y^#qtLFEq_|V=G8JmU?8LZYU9qczy_qbo3WK#gYmosvwqJusw&^6icpb zAZ&uM{dljofQdy}{=9@b)G{74#1>>>@fgoi$%Vzic}gW$l&BkojV#@YD+-oI6p3=Q zFgC=J3MB~}S!*=yR5I6eMTvSt*jyZ9BhF)l=`>-ZtvCHF*j&-Z${SgJj4)!TCiNzo zwh;)FXtRNp3pX1s+|Vu>dmY4@#AyP&i&T=+!%s;-L6U{LK##Od^avUaenl+SRgYS4(I| z&l9L7jml_x*|fPN=aV9Ul^*%LXuSDdtVtS=rk9us?R;$LQT(vB^AXx{=9;Acet765 zwyT47Ry(6N&G)nDy$;&-qZeEYR$B*_HeB{6jsdcte%MDXV)M1ZMv=>k%~!w{V8-U_ z5F6Q83=1=3BX4Zw%@5eDW55sCV(>F5kuF(dENf>esZf$eB1O(v%Sfz?cCkV(NU|r} zjKSt&5etp;7<$#VAW<5pG1!(Xm55$PbBVlH3|4=|d-a8OA(oPS?9>_uc7FVYaP94>4HWLpc= zp1|mKC1VpJAwaIKOVBA+YrMJ<8!v0*!c?>^-ZhSV{G&4!Lilz%G8O)RbEcAnk>bjc z$EPzBr*ud#%4JlzBjz#?3=S!Wk-k;PV}4&QQ{g>Gz$oQ172b)KGDQdZeIb{T-FJfq z<@ep;J>~7y@QM}cL{g@Z3wgo5*LhgW-q2%-tXvCTM&90s;)UlhOZz&A3hUI?vM(&B z1NfKl`yunJ@J_YxkMX9G_>*!gvI-`QC@Ht9XU;!LR7sZHsve6X@eA>t>JbGwj7B9L z$&i3Jx=4R1mMBX*8wXpWByCYm!KQ1erW@`_5sh;76ib|x&@l4x`4%jB2uwV}uMyswZ2jg$OL9!~O7({Pe^nT3*I8y(e=*QZR>;Xg3~T#1{qNNe3BO@lgCQQF`EacPMIk_W{^4~@6M8bXi z`kD2`8HqCrC(W{BaK_@K_78E=_y3n86>q1}<4rmoc5KtTfarakk8}7lcVUks$&2iW zQ`4vT{03kBgo62_ylwl3v`2#EE5Sz!Z^xO4ciIpi6D4r9zp<^nhMfj;HUsd05kvqAOfIZuCJMZ005W}0Kj_yG^n<)ovpK}t+T$0 zhrOwjF1@>rHDNwDC}l1H^zHfo9sk8&U?6eIwvQ2R@GRj5LA6>LXOmI(eNg`kE_-1e z)G)R%$t-C&d|*osjzM}c!Y6^1B1N3XM_&hH#G#`>Gn!C~P8!jk{V+J4ohFB$N}cjK zKe^+Gs7jKq5v1xsP`78U7wp?%K4O`3(~8QXo)!(12a6eg2jf^zyI-Q*TI5I(RE7Nj z#kqfNYr2EyUy!K<&xFm_CqqYPDi3_=BnvZ8G(lYL{aS%a4GyFhm7!LbdvzRfLx3#Ti7ZpUYdjUo}X%9W4qI6J}G4(^KikbhOziiHSfw!}wwthaknDk@l{!CK+Bm zw--s!vJ#}UPE;#G469|2Z!;%AvAD?3F!sh<@=hfBGrt4U26ya%hc8w9cw8Wf4ee%; zpC1bEu1*E?Llm*w{udBmD}t@fNA55_LN~`xa}~9A3vAqowBT`Y-t54|qF2`+RQ9gR zd08yj&LJ%_PBvF>;rIFq22l97;A~Q5B)NL?IGMNDgMAB5eMeJkCkFao=l=!ee{m%K z<<%?WW##)B;eyX3or6y4hl_j6gxOo zTAC+4vOyv?cOg=TG)1qD4fDPTHyE8a&0lL!R&(9xt{P-PNcp5HxVD8o_b_29!*4OU z_yC?ilvCz-HVtjq$;f=U+I!fV==vU8RoQ~mveqcuk&DtHoK_XRFZ20i@Ma01X0Bis> zh`TkzKjOsA&Jk#2X9xUM!~PXBAa8~24duW4=!~DR8eoL`bQaVdIMd}2|DrI;R;O%Xl^Ty@eC={9iXCzf#|<`!UI>;m!x?2rl{R}qUg1DR6;HC&9( z;X@q_>jeLequT2s?B98g1_HJ~*z)w(?je5#u1GZ*D1U~bp^Y}5AsrvvvAz|? zver@|5qeycI%+hmny*9;?WrpyCRDN&k?8qGM4pE4e7aw*h8EO@_ll6F&;cscB5hw? zKGsP&sVTV-S*d*hI^eV?OPZ3kDao${F&crb*5^_%NwQ?rRu`Hz1u<+fv1mf+=6ixB zEsNj+T!f{?mnc!z&0+u|+z@gG&v3l;yGYdZ`?E%p=dQ{e5m{ zN0-9cg7g4qp6&`)&vo3a=?X)o(=uu*4LFLHxf9*)J<@RqkK`Bv62Qq9n^u!)C;^yI zG_iC@=d#nX$tp2)qCR~z@R$Zasl{stdBdPZu6~tb&JD)Qx{1olpm}|n6iV*_7Eh^t zeFJP+S$alNq zUyL*EL-P0^ZHYfH%z>F}UyGqNG_8p1nHu5b*MWWRyyD(!uU&IBkKvi~2>mu=?;QI) za>oS~66R<5iFHAt3PLxaaO^0q?bHXXVa-|#W53X3J;HYl zZc89}o5x0Fs+Fg+jlJ{43Eb^hC9T3~ShK_Pp#cYRvB0JWwF6s7>`SCBx!{5@7R-)j zb~5Ev662w48Sg3l!MV@p@dNB5_Eu76+1I%rLMsq&UZLGuuomDOgAc-O2~r)}Q}91k zWEtetxy>W&*1?$iJ-(urAevLJlSh;c9kh+ktdT}fF2?Ns+&>e&tYY`~!)6bj6~MWH z=zAtOy!Lwwng2}3eWJVElHdS<5jp?>|Bs+@a`vz`b^4WtziDX46tJQC*37?te44a} zVMGe5WJ?HZoNX>=RW7SRNFC5JCb2h5sCwA&iX<{5PmmE}K51ltVY_kbcDmrai{z_v zGUJJ>&kqGoQ>MQ8zEC|%d;xJAh9%V@ z?gKtqK-g@of%Wu)A`9-GNKI>htIZLK;^UU0J_X__+duBk>Jhk+N#g0O%yDa(+ zm1Y8~*!P3kY><{*pW7)!q>gH2t5Gz}K<2LwQQ4R=gomV5_i*dj6F;|$0R;9Oge%GC zhZ74CD0aAu{64~Dn6j$0w5RRaF|kBt@9-aY*~0UuVfq*&JN>sknmYC-E~|cm)+8@0 z3b(zYjzVGHshcRg2U(pGO%qbGLL3BKeO2B@CQ19mL3vkmX~{Z}eXtZJy2<^p z5G*#R{jg%$Hcf9Twog=W4!)Eu3)b)|s(0O`^2r(qq>_EAgF-{{Gg;M8zIV;S@=ls^ zyoadA`xG=9eL|9%xT`pngkS1@ly7;AP>FNZ_u=cZT5nq=E+xf;Ef@yGs?4GCf=Yz2 z1(?FE3!Z*%*~9#MV2oyj3#9x`axlgGZ5G2#ong|}J&JY=KTG>2=S+E|xB|zuf)A=> zdZ{O@e@NsR89n|;`@)iz`CiBfa>x^jNpI7uEH~b}7{txK3I*lmZEBFBZmAdjx|d{2 zgrbTB<#8OQMpUwAkTf|mP}LVj3MSH6w$K+}zTK{DRIi6i@IG_021J9+3Q~O+7@i(% z{BmY5i;zx;3oqL>zP_s{1D_P)1(yEnAD8q^PaAaWyhT0IP2G?ww~^04pDx}0Z33Yw z{ChCRuRmoLcZ+Z0K^NmE*Y=ht%)LUGRM@G+ddhF`jsTPDiJ$52le7^HijUTKN7ZZ~ z%d{M>;ej*|OGTvpG~@&=491OKFF}RQNr5LBsy#ClM8HtYlkA@Iqn`JaSm%l_O#wj8 zC!C7-!TqVn5gXR7oHjEHd7(KRq>ad@k@>5xjuB_B2N>eUvHxw(`HX?oE%#Q$2MK?X z7=ILUXA4ssQ-(jzOusVEvF1bs4m)ZO#sfc`liSmW?HIE4%_-}IHBytTMBJw4V?{L< z=J-yW_n>5)*GiOyMJYmdyos{{VDMd6F)*Z!2Lv+|6H6pjJER%bQd+L?5#ldhB_+4n zzMemwFJ`+^J;hUD`{I-JyA>{ZGb5K$N#~ONfE<*p-trQOk;tYnw)e2KPtvP7zK+N? zBhW)jVD}={dC3HloU7F6U!a5IT6@hn{5QxWha~z)a?pox@dDRV(P@g5sxV37@ALwU z6uf+xiAav&lcfB-$~xu0GsOwPM^bL6m)DWy)ou6J3xx*lE1lvIty~&rGjZc@dClHZ zqM-O%+><@;VYD!y&!~}$>_BP?+vdJ9x5*fIYs@%Z9(=Z%C4>&t@+#5?E7cjgK6=*b zX`U?sv0x2IDsEolI!dyQHH24qSjg*8KT{n;U!T_3ZD_+Qj|p1LRq?A0CV*?-6osu3 zq~g=*56LnsF+W<&Mb+2tUz9%4Q)syYDu_rrwg%G&HCef}t)Q9o?MJ^~hYJ+!il>@V zjD@c_R#KMHAZFjuiacfJ1AGKkB9+`4`2r_D4PDTjjGo**t?`r?PU%$Ss ziCWqgicDGgbRkCn_4yfNUBBn${P2SPV4mWlZ=mPtek`Tu^@8iVcIz7&L$}Z6oDlT5?IZ;^MdG&YWGx7*Se}u;i5dN$IMxlXt0pxwx($leFP(5wJ!#?kRSRKSS<49LbH?Ga{^s)1E`)rxDgr99;9bwaPz*Bmu zO2F+NORoBs(sER-=`MUYN?0++=UL^RI^jsXiVoq>okd>eEt%S|t5c+!z|c7|VMhZp zCHh8TqtLNFQ^HRxmlNwYHT|W)1zv9x62rH3F#3j;?F^1t&?_D*af&c~!0V1M`N8by z_gS30nt@crGo0$@gE8$V>0PdnYiz-ucW~PXkNCO+Z1K{L`?GBkm@IL;u1ir0JWeYz z@xueYKGuJAN3HBE%EaC7}EOIqo`RCbOW#Pn+6bJCbLWEtCQn^7k=NWwdos-zj7RK3nL z05Od(2Y#h;Oo65`y}d}ZsWrEA2yjQI3`5>1yvG(%o!{nT66p2pX%VaPy#o%f_09Lw7Zi}0r9W&%kO7{Fr~4E+uc zV=W*Q?L|iRl6i;rW_--IhkPtE27mtZt8I-1b zBK{39w*<66PmAntz4q~yPG3D#o%9y&ULVWN=Jfb#OecY9KeYz0Z>|Nc(mK8UN6F7F z_p{|Y>vmX(5>LkMmNQ-TDEHGnCTdR3ysff5y$lv2{@$HRCrUU)s(4?E z_iVjt{>XP<#Z5B>l5%+t(@@3?N6`gSQ6E7FRnPpB-g278jI+j!!eL6DmYBoNJ4PEc zH20mGP0_BM5Wy9}U^CprJ!7UWcAVx9>sWoaM^Mhq;3G0w`%?G7GRE|%q8 zqJ?xMk(*cFOgKH#8`&HnPw{!^zLH-PyQ^ zaqwPLQ}7RGjcd6G&>l%;-93zc>s&B{a`Wf$wD;pEQQB7!vP&VMe3-=75x0?ksWBq{FF7M$nYXF?id zgB>a#lWxGRRY&vd-!d>dFLk-TZv>_##@ig>ALaMgKHAB`)YRFD;g8E7H8?U})^7C| zR|R%M0JjFzl-n&5%rvr4sN{wdZgDXeEjA{K-;tC`I9`m2KcTehnTBvp``=7oKwe|3wz%_0y40ykY9{avN4t4R?GNa%WwmVZO6!DXaYh{+AxZx zNgi3;D=&q2V^%x7a3?>0NwAi;P9f#T`wYJkth_N%)_u*s^{lD{{=xmT04Tc1?}>7c zHJ=SXWCoX2MlAMm=;|e2^ zH`*W~QmhTA2_uUOk#{GAa9S}x3Ri0J$?B~MmUauxYHLrtjvB>T_F3cCdUx&|yQA%s zRxLe-9`Qw5=9$tvk0;6>Ef_3u{PmR4VkP=tPA*?O+qwzt$aZCYNgE#dUH1q!Y(&ss znGPF%LbTTIiP%?PF%vgEUxAiRj2#6=Rj1lU~sTgs5YSW6AM4C5PYGcW z1GG3r7RS(+iY*Z) zJdbVbcSh?cdI~q%W9ef>5*2nPc#&EbRe(#SPjO;Z6Z? znQTg_J$63IQd!X>JXgd}>G=V^@oP$&SLxMis6cX|DR}BM@`gT|+uehzWbNMAIIju^ zyee6(vi9mWr(KWqp=B@uIWt*E6z6ud)oZ4V1s`MZx=8S*A@&N)@k4RnK-sOs7PD6# zU4bj^iqn3s(BU}azHqyW9F$*a;z5H%nM^^&MjiNU>}Vz#$at|-fr{%`;wb9MMIUbn z0(GJ4jZJ)GukhfirSR&ErQ*2Pob{hjYU3T!<#?+XlTgASrpK~?fs|-tzUGk^mnP%I&TE_%` zMpQkOgpB;PEUnbkIE~t{q%2LHD)Tgpx)SB6%+$v^=}Xw&{9>thlJ{g}_ic=NVpc{{ zR>+~|m4&g%Ir^a6`>V_QDSH^E80Lk=$xFZJ?*F7|_BqXD1|R`|c@zKu_3eo_?$gP_ z(9zUH+1b(3*8C5)rcz_oc2x|?hhf?evlOvOQL`S86&g-k5I7|HksJYqOcY=J$FR<) zZ>DZ3^+_eZviz0$vn|zsDelLl<)3ThDc-uHx<}#+4Qt zSm2bcv}?P;dlfkh+{DR8NWIl{w2|2`2GxLLBKYjJaPbC@L6c4Y<9N`Q?)5ueMo0Nh zC66M|l0-^jSaO^APN@ssj5{F%PV-GBy7Fm{XR$rbuI}_P?qPnbc%|BlBo&Azg>1{Z zYM+v%i0*d95n+|2Hr0bq-I-Z?3=9wK{PhG0dM}T+{I{b38)bOoLSCv%U`#7glp#i7 z4U-_3=;*al14*wH#Y!v@QMi{txx+55y{=nFxogX znSf<%Bqg(Vp!%xSoD|KhsHJYIGp+>4?GvBWW@?csWgyGZS7V`}SW!c~B4l#>b%j(z z#b!A`S*o{+c?2Zq`X?gG?G!VD?L zqCudPevU@X$yAPPJ}F}07dI%x+8TolkP62pTXoHar5|na$wG@ zpkIBd%rqq0!5N(T{#Y&Vc{vA(>qG8YF9dlcjLb9426I=u*{9W!7qFcl^u8xQEZHGG zDgX%FrtX7vGoi@^UwW7CF(sPUr59-(GXU)Pvfv-d(;O`m8qUODSQ=JjYZKClO?g61 zJ2MQIDjOTbQu$^@`U!LkvsUv3?Ch1pl5^$vExdVr8w~|p6JG7k^GZ^+a$v@z!`yiK zS_XD4By5zrcdo3jM5^1QVM{LZoRW@$<{Vwh-euJsja@8QvSOK^+qb6{YjRDVIl}o; zGbTm9gsgJ>XyX&li*mWNbxOu+H4Jx09c-8EE&MtV6B;EaqFb375mkUQ(Dme@@;k$`b9Nz2;P88D40Ew0U?#b{JSqqgIQPL=cLyJdfbyn5rFcxzL0k6Vc z4{X6Y%hhnV9_g$c4>8!*UAjVJFyQb^R;T-i2*aXEsN*{Uc)SG>1o#`A!bvukZ005- zC4Wb?)Ypu?Ztbs~P3+Hy-L-OlkbzrwUPoXM*DdmxgIY@xMm*iQPN*Pp-v<4enHtFjaeUyV^CR| zvswkk<@r9_&W+Gy0)$A3#bb#DyzAKLS}<2uAz)Iw%T!v)Ks!zc3z|_Ls4XhAI!V@z z)+)HjiXUw0Mac@0f!XfTzns|G#)sE zJz=$08S%c|$__+1sH9b2aK5B--mD{yOyPkru$>k-}A{+F;V}Ub9*n`$y~zj5TquXKY7{1l>MDk0dF&o3v8EJ${ATcJP0(9 z&X@p&6vy-a^l)Umzu;^Cy7&{f`}N`GmQl7Tg{Z^UGlQlY>TJK;H||)k2}syP!=jiT zYzZ!4w9OGU371?Z=)Csl7AX|^i;ljJ*SkVIac=a$^@Ba@hOle9fwclZCab3x(c^wF z>jJ$yWbqEVX^j4%@k95=Z>T*|6EAObz5lFw5$^_F2;YVi8F2sSPKEgwTNtT4 zX1mIW)QPdr58WxAXd>POKPDJQXhAO;*FDMFR1ckyFE#7%bgQcrPvBzsm|uj?(!kx2 z&fiIc)#YC |3{YhlnUN_uKH8zJo`;45lKKcV$Lc4#!ac_Lcmm$QTRct_QuvTa+ zZ0*>UHdnE-oez{|#%n@>cGNbgPVPXp`DREBECH!mT1m(9R11kAMwS-){c;rCY}O4* z3mtJI0p^hBRjo!>1LY@lt}o-BbL$ROT7??#sJ_w1#!Fl*kCo2^(~(U+XGdcZwDwP6 zb)kH-sGz|h!8^g`&GC>bK}|#dsk8~celskf8as7x>>{#ng#UC;W&n(tj58iPBlPE{ zWFklLcudi%c@AN?qZAaw;X99vrkWa%%ky&>1&U8s~#D14u~fLnuvYvU4V9eknf#>Fh^w5WwWhNhF!Waj|fs4cIrZ7 zpwNDmt4YPY&_KlWry-u>K)cF0E9jE}3<5(85PZ*Mp2?pM@dOn1K_)o``hD~XJbdrY z-+4>0cEwLx7Pg}M9EEuEourTpS4%q~G8KhnlAAl}lVBPBr12Z3dtT%{K2xGua(+Xi zFU%<_8TnrAT8q#tw+=%Y7a5JaI-vb5~# z1QCpXv1FoE^YKEIXg+rMHlVLFnX~zAl&~cb<{IzkY9w!$}mpFi5R6}iSv8GW?)h9pYCCNFV^{~m-iz(l{S{O{UW=ay% zYmfA%9Qlnrs49$W3;Px|Y!m=($V?RrlQzja8zz$lw;!4|Qi8zq$1?h2cvhv=@}w@o z_(@vq89X=>%^%zr`scq{?9(!2;LD+#5}AC2(cAwuhF30E4z(L}m*L`d5F;y8FBkJn2#hb44ZsDPf!IX_p)YoxBou;KK- zA3sdv#b;K&wb|@%IS2Vofns83tl(&8@5Eqi=Vb%uc$b5lSxEnP`m>EAY7b$8$Oam6 zGc4)pfVsg_wPVxAR9`bmLer-5%r9t1jJ1zjyK^YmFq7NtXV_OJjHHSqkT4VBCt=tl z9v^wXQjZh&i`6=JtD#EwY)1vCHwkB(FXPq3C8`y=Go-R55OYgerIKa@)%-=#_KADw^i{iwqT6xMye;oRs|R%HlIapT4KzRqSxH>=;DfY!CA>R^j^v1wCU)-+vRnd_SE8F+Pj8mz@W|V2 z8hGK-0*`sgNXh8D?AWuPHHfR?YCkmTmEwwsH^Q~sG=`6F)L%O*?)jSQ6bYhU^c}Lz zB2+?cvd8YT#xXF$Dg2F=R76{yx^YhWRp6zqscdJ0QSqYd0v>Z++77?`gV*_8iKy=9 z@KXKe;tp>vj`UX-H?+6^pNGG>_CJrzguhapIBm-caQ?^n(#%)22A|_y&E<-X^O{XU zE92fM&S=F&OGGf&Z45o$Et<4dV`U0FF~`s5hn~sh;9c-RKSN}Ak9%}|6k(X|Ko2*W zlqpDEGW?zlnwc6gmx;)Tzf&orIdx2@TL-~U%dF^enTEpG7GC5EYjQH$pe>6%0L>p>2Nq)?{-+TlQ?RX^jm;}NF)0)!%}LCVN6z}tWN?c7aBcHhbX0aI0)Z|hHTIDNHnZXf zJJO9o+94s=Iqk>irP$2vcCzUE8l;?3ArodT$lxu`%t<-pTh5#uDAHEkV+cqjOC91< zyhREAQ96+qjEgkGO+0(w>NB;B>qI+JU4osU*7ZXZ2)MITd%cM1PEJ2Cd<4uQCb6;U z#!Y!8?1HmN{u$v8MB*q0dL8tM>VVK7O!jtG^zklCBp~uVwe1y6+xSpoGTI#&Is?iX z&hxe>uCIcx4~sN^(WMsohXVaqsh2mR`X9|92q@hf{qgr=_<#M=e_j7Z8m}Pz?*RWU z*!`E`uWQy@;QU3t`@7-4ix~fE*!M=#{NIws-*JAIwEcG03bpI0Q^UP@VoiHXWPG;Q_%j!{GYj3 WK^o%CM}8^9VF81`l;RA(?*1Q1J*>t6 diff --git a/src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.csv b/src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.csv deleted file mode 100644 index 351a34211..000000000 --- a/src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.csv +++ /dev/null @@ -1,3 +0,0 @@ -Germplasm Name,Germplasm GID,Test (T) or Check (C ),Exp Title,Exp Description,Exp Unit,Exp Type,Env,Env Location,Env Year,Exp Unit ID,Exp Replicate #,Exp Block #,Row,Column,Treatment Factors,ObsUnitID,ObsVar1 -,1,,exp,,plant,trial,env1,here,2022,1,1,2,,,,,1 -,1,,exp,,plant,trial,env2,here,2022,1,1,2,,,,,2 \ No newline at end of file diff --git a/src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.xls b/src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.xls deleted file mode 100644 index ac5075c4a3f4aaf137a753f54e3641eb05e7093b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36352 zcmeHw349b)^8cHhlR!wg!YMQmAb}i^aLJVu!jS_)01+@uk`5V2X2Q%QgoT(a1O-Jz zKsi-FkpOZFa^IpHDj<dyo6hM^>t0#8rRi0sIQlYI_FIJ)^N_(+xb7#F{TS|7R8Ud+j>9!THOK!$67W(( zksj{s>Ew?+k24qMp=RtVzBjua!Y5;2;T*&ASVP7VwHXGzQJ-bvlC!e389X=K$Q!sn z=!g+`(5Q%qgD|{4DmSKrO)dw#E^8w2R!{h>FMMtiYVH+2`>=Yf)b*&_+xj`tJRb_7*R8d34y0E>Zs!AoMVqR6$aAA$Apu(VeXT1$psaQ9sRNShp@po8+ za~xzJeVy__HM&$;qjwcGR4md-UplaCSZFs34j+Y`ax4MXI-X@Sli6lfuqNj^V$9ou zPiglp+?q3YXo$5AXx%2bO`A?>LA}NVN`)~&E!@(n_=&>j-^ViAwD)bfv}uX=}?XtO2Zw zu^%cp;amJT$PwJL2L2zF1Ibp!IQkL~7loYPGsE59w54l-Z>j`7yAt@)O5n>Xfv>9s z{%$4k%J_M$lKSf_fmepVvie&p!ShG+vrC|hR7M;x7QO7R1irlz`1_T>w^afc`)f~E zIqF!oQ;8|G*S{`W0+(dpn;X?#J@DnNshh(2YmB525 zfp>5SuMM06{_oA`nTX#0DevS>nWlLmiLce@%;rnEedl5&RDs5*OwovNxI zW?R39qx!(+5H6!@=2N7LPX%4EU(lukj|gwVSfMRl3tUlO#LqDXxy^Q)?dE4g7w~ar zxQ!eF1%B-71MUz%;11;};t}k}hOWH6T%K}%pnR%8RGn=Z@;`GYI2Xu3nm7t)pvb&%<~*f*dmkJUh&5IhD2V&t(X zxIJzI_s4BO&p#fECC|rgKuUQm3KT{js{zHA$7(=<=dl`4#CWU*6iObe0mYcdYCyrq zX@h9Rc%Z1{TKRZz69x*OMJ6ZCAULi_twchhsv1bh3@b-MajhCi$TTZQLP4+^NXT3( zM?#UY8c4{5D@Q`%vKmOptSd)Cv7kyMrxKiNhXQdmkT};4(v|8UajqQ%yXqiut{o&i z)j{H1JI$(x#JP4b@2{GZ@CfJH!E=G?AaSmp7S%)ITsz3zs>6wM?QqpY;#@nes)xk6 zcJL@g)*TdVBfaR^7;~Ue-$yuNFF?RU_s((MqFH6T)~+SwF8)! z3 zr+I-gPY!3Dg0)XeI-!~v_^1J9r?w3w8$0%`+{To-4IW#{ z+z6@EhAP<(0iC-MJgco5Nkyb{BoWm?q6{}my>`2ulbu%d?Y|eS=;$4V0#;DpC(ugoEB_;8yuuhUBgss`BA0g#&v|`dBz2bAvK$ z{U(yj0@; z+fq@grQe^;K>t6Rf%re0fdPLu0|WnT1_u4v3=FOg0~7|7K}Lkx1{qtJ=p98EnhS@0 znvno_HJp>Z%eEyZ#b9F85mO?0ai&A@o9ao2edtZZttwr%h(wxLR>gqClw+wVol{bE z82C@deq|(01_QrKEq`Prr6tlT)$&J5WoY%J^2bI}rK(bXFH?Wy2paa^R+VxDrB;W5 zKXM2UuMPwMNn=&c(no*+C+YhFmwL<((+d{fsVx`6MQ3F2I#cKQ>O?l59^IJ_m0ODn zA8Zrr#M&~Bh2V^48TjLwPS_qtk0eyAHuH4jvo2hHP~lG3lz3x))0P%PiA-H3nQ(hD zt(}k&pL_^p-1)3uuYc~COg$x;2zxT^oRATpjR<5s_^i8k?>Z(^UrDBmJ(&(p$k5Xh zv(P;GtcnMx9Fu9FB-7QNOsEqw^vuOfM$Knky8X~GnTAR--R#MPJ0T-JMG=JN#b z?H7*8_!F5f&MmDa$aJZOrJYK6@7Onv$uv@uabRg}oRE=Q+Np$}PHu2arm>QY150b~ zgpAzMP9$Ar zl5t>Z5l+a+E$vjou`BO6CexhAbaQTL9LRL5hNXo^%w1cFZk|w*abRfyPRPhDEj;4; z&83ce(?Utcfu#jHAtSf6@QCBbk2@yQQc1>vr3E`7Be%5hh>8nYj>&LJG7c=QlM^y> zOAC)!cKB<@WLhc7IIy%XPRPhDEj(iF>A8-`)W$w*ERD@B=>2<%Red$9^ha-?fosx)id4;$&RX((2claZgJ zF=_R>`Rc#gB6RVusKvzJ(qyoF{GvvK)Jx0sNWo~5MzNQW(vx_J04-w*1UwkFq~Rtl zqo`0>UQ*kXylhGfTzv8K0BnDX&E_*Eao8+bvZT~kzlN)O-P9w+O$)&ajQ_HO+J3eWDs)S3^v8mBDBO4EkYG7JS%G3 zwXmhtkcA-wXZ;HEax^-PNpG0KrSgTQ`RaIER4!dB{P63+Lj0DDbgvWiuLmPjAoEnP zQwV#YTwl^YDcfvbUg#TE1mpF37-}zDysD-ZZY&-`(<@As7BeMWASU6egP&jEIm`fF zld08>=P*}LFzV_D$qoQRWcyk$Ivxy)YoCP969XnLb%KbFOwkjZiH>lAOC3xyWXR$B zl*sEkKrZ^fN|0AXuG)DO9UpMVB z{!rAIjaU}Ac(blhBpay9%ZDlC+p5f^0s1o#lFfrHlG3fipB7IgrO^uOzd8K!vVv$spiUNKV6Stz%O?+MqjLgSIT1?|Ct^wBL@bfR>{;Rt*#+3L z;O-7Z3_+g_mWe-!j7+r`nQE%mlmlsXwr%C=3-zSJI!sU;$;;l_RfM9qfI@E~3s5|J zaBU)>Ac2e8cN2`t=J^82UGgJhZ>18MSY6f!j)#rZ7>HdpqLWFC4~7k{Zhk}mXi7+9VnTqUCKzC-Flh5AbHPsIc;1kUaAwTqk~Fz|eIdG;dDY}_ zalA>R&B3$xB>g+En{1xbp%$ml;!L<6Z)uy(leYK*O-{bX1oxAn(Q)ZKr`2WTwuJkc`o!CZR2N!EO_HYbNmN=~IVxf)YOHkz39LX%9$D_vvcx%4R_?aX|GK+7Uh zut}A;{Jb2!CNq>n@2#DR9~h@*s!`CX5kF|;pv^4SjvK3DoJ}20YveSXaf%Tff-fC! z5W11C(@x4qH!`6)ZI%{40ngGKZ1^^1qiY&2&!8V~(By`4GPJq*Mia*uYVm3AzNIgL zaWO&MWUVQi1Aj*J8fbYg!@wh-gd8=Tj-QOC7NaT9YB+6fp59;zl@TCS1R8@NOaspg z5k4Elkd*{;8eJx*hl&hbrl3J1s6oW`;&=H0!`0w!)*| zfvG_p1SY1Wfw5eW9&S^xJffz}%Hj8kd>WKZeSLdj@)Zv zuoWn%&S=cf6>SDxG!Yu$HE4NbC^BM1?J@&^7KZG0MyZr*9CLDDI+KpBb4wB*J(}DZAPdx)MO}Nl#pz{ zLJh_eZ|upz>`lgCd}(y1V9o^Tpp>mG;PD0Ip+Uc+e`bEjc`UT!$I1(|XhIb^H$!*> zj5$}M%YZE4gK~mlGJ2zt>^eP1p8?ewVXZKWjBJe|)7<~ZH!-B9{d5KQn3oIdk3hiV zFGPs*lv{J~-V_=p;hPsmeI~x?0XU6+Ev^lKW(nB^C)_Cj)PS-MUSF)qvB`j$Fk7M! z!wFY}i?a7T0fW|3HxzJbfKZiI+3BQ36*D1IrDN*mmjBv6r zKO}xK1ei!soEi{MI@Fsi(6yHX@g(w|C&-BUBL!${JIDM|9|(WGATvF#Gr?gx5{OKM zIOGISH$u5)mBFomOt-_c34=3>Z$acvG((m}z&8SS{HU?+I{P)Y+1$>~N4xC7&oRCLW73Pjoj#PAezLQBq8 zRaR(K5`>|bp%cbxC^s@211=991r7>h5NZ_@G@Eu(wnjq~*#Yb!6L~EuLdgDvye{F`Bx^x@H&ZZ%;vvBh;*#qOF$-6eCIr(oA{bsvz(aJ*y)sV| zK+7sNW9Sx{nb;Wz2aDC`=*Lg7XkD5hgmTH!Ehs$K$O(Bk(!xwWORIxC&3!Z{`{vx6 zrVfs$6_kq-ui3)XN;y5TutCXqD2Hq_by!bx-x~@hW-+T&7@pCDtj!hXCJ9y6W6mU1 z2sO{+Xtn|Ek`~$nZF^3XIr?CZMv^I*r)7#BO1wei5@yVD$g*_Yz+uTfx^?O-Q3xD` zBP1~~BrYyBJG*CYZcn4JV|I3KE{z-q_d@8Nc?tuqV|FG?07*`wW0Dj`F-ma8x$sx{ zpsm5Sg*;3VL+e6RG{S`e-d`Nxu$F#OvX18zLQ0EoA&G|9 zBk*+=zrrh!SgF|s>RU47EQ_G&CQW#t4TCdrdvE_G_{`K66~ykio8Q4KUp&a4UA}vhM*rZ(z+aS;j1y zjgE!*G1Sv2hc=E4EzHe%_mz`LM>_fSdpO|N88<=(E#DfYZhLF`hjTw%HSOR>O9K3l z|847<;lDggIGY+%#`)(2Ux;}4Mz3pzeJwq=A6dM7&7e&WVmb!gZWcdw^_Sg~FKiew z`>#zK_tvcTUHtv-{d)tuKVLF%_R^HqQ@_p|pRl#h+p6YY&#?OlbzMA*m?#OOQ?+%Jp&)9Zw{<%A4KleZG z9}{|FSNk7A-|jej<;SBAr~cez^UY`Yu4lTIUFIItnp5w{=GwEGhES8O}d>C_v^hl)1^-{|M2>%K5Sdp&(+S=hB<`{tGL>&~?}y?y7; zpFcuZ>^nMf%4_~J(7_jQ(r-sGMsoQu*=Hiw5-{T?3#ol7(fbJJ7j?C0<>Yx6x@YVO z?3cdvnlrJ0XGb(twr_T%g$Dy;{zdtQ_O|!Bu z1~mU5IOpnz4Zprs*!cK+FV+8Y;77)xMQzIG&D^m468HJ`j^(Ys4Zifuu7C-BUW^}k z^RtIRU-i4)v7NU|568?Woy|S-DMrlZmWdw9` znYZgUdByd1!}eQp+N7jhA29G-XvjT7bV`A#U=?UjK|SE!nPf3{@? z7rpGaS#@@u4GkLloooMB{CBX}!9@|j_K4^e(QE2A_qNUYOZ&yO4u7>fZSU~=eVY4T zm~}MzQt{uvomO!1VQJiV>Y@JEc1IpLd|=N%Z?&I%dPnU^rG4hT{6V*MzpPw%{KsR@ zH9j0(@BOEyXZ)0Vs<1R*%!6?mN6!S^eCg(~td&D9%<`LE6xP3PS)b-FANQMDwEFCq zqXPBAcFtKnq*m;Sse`)i2;!w>xa0D@1=HF)M3w(XMI)}8lK;{qvPE{ z?z93ew)w=8vzzK%+Av|_`s}Mdk4AS~KKbN7GOl(FEKhs-(Bz(p zn^(;|zoJ3fGlv#6+ur}==bNH~_HUiO$-iLq>UFErX2cBo>bUprAschLHd^yBm(l;> z^7~KkI=5E$$*bQ4&U!9*_c{Lz|C9M!{8wfi>&1pdy$~?F&y$1iH}+}u!m?TORqZeL z^@>sNb^9RUph2_kbWqNr;va%;>A!d@yIGsqr@wP)m(CCT?z^M&7Q}u2$a6(`cTWH1 z-Y2L2_OClX>&J&Zv%F6QzyJLa{PDXhdvv;Sx#FiEzWlQI>Z2cS`+l4D>;m1sIgQ%= zcInf!{l7eV*>~;p^}l(i@Rz&mCvQsI_vqgG$l|};I=%nonS-Sd2bRv9T#~#mK6tuo z&!4WT4lIw0j5|6%s@2VcGTq73E-g;^bdtgc6Qi-Endgwe%7b$(XA5~)qb*mPVl}pM<;S$bs6|(?RTS2%=~eH+p|k{si&_G zxoCQ`(5L^{HM*Oz~M0Sz7yA!PG^cv~G61?UJ3Qdu@&-WWJcVwABms8tsVx>h#ge zjq0n1UVozVt?84O6g^)y@xJQ|=Xd|&b?CJ{pC-G#+-ld-g%8-!0|~p1EPB{!+gI;* ze=DPO?5M0o@goW@-8kChiI=qFX8zdsScBJB)jf9k+AnoHd|rF1+ouB~)veaw(Tp6| zC#vAxb+;SW%3hF~wry$ahqE`w?0DXF$8*V(cPuQ-3Y_bgw&rKHSNFwUpW&Y@8RUB_ zFWtpg{a&$OSkha~y-#c09zMMxzSY(Vi+20pD|~&@$>K)4KD@HO_m1o_SN6L#nWk+r z;LbazJaa#bKlcUaySVRkk0Y<@qK_JiLs$J{o&U0)H;(<7xNMKAX`A(5&K@vL_s?zT zeq31b`i>7Lbsv*_H2y^L+Vzbtmw&Q1DS1=GI}KH*)5BVG4^4@a#!l^>zq`x1 z&@0*(OY7>odF3{_bY-?*i_&k-tQ`>+b8YMXjWq1oAt!XH+K8^T-knd$b;{nJ1{6~;Ldp= z0f*0B>3Qqz3ta1Z#+T=p-CKWcJP>@+YU$`n_3IsuAH2~+dn2QbXVUwx&-pN;_riBB zzSNMv^?kkYX-hjUKQQ6D^)p@#ST=pn)T=koH9H+tT*@6iQNAkt$}Dx^q#+3#o@{-; zUGU({DZhp^d}qoRUDNvAYCPzC-NcwKn*y`KldtxCb-d~O;aRTNCOn(fsZ29}N~`8> z>kcmMzkQu{MAwu$6W8DSIp&0+|JhR$-+6U-YPo5A=ZVkvYx30A8|5$TJv2d+{*g9o zX7aFyh7%h?43|>goHyL7Q(;zQeEGhp=uTxfmW?{s;97Z?cMU&IE8qCUi>`0P-fi9Q z@JIi8b>Y7VesvdUw=J1w_A_ruAiA6I(g9Q(;G`K zgvOut`@xhvf3;7@?tkJzhJ*O+b7tO~J8-8NqI-Kq2o#eRAt=hCxH*YQ^gi6FLFvQk zlhu0tXS@AgX?JPX=?&*xORsnJ=@uC<^v+$Mjlt^B=QaD6X!k67{doP1@XMXE*ELo5 zy!Gm&rH68N-kBSf;-578La)rtmf3)ZHfQR`-E52K%n)X4!zI4B-<*dnTLzY%-d(@-p?D1hO{m1?` zG}?G!rhDM3W2dgPS+*|0?ZoKayN`Zp>AK1#1iv+`EUQH|+2J6^9WvM!filgrw(bOZ z%91{1LuJ}zZP*0)q+K2&6TcFZGKaBZ`rHvFA)`^^P#S?1#|6wIYYvVH$E0A0FQX`;bZL!MCSd{9o zxN`^-9!Bp+W;2A^rNU=B)Xox)uOsPorqXqw13ue`st?1p_I<=%ZQ_aSo?t3`eR2R5b9LA@8Xp?0AKTe`kZ2=VUbNJcQypf|gT z38xiC6};Kog8oa)^{9e3D-+r>m9Nm2WbMsT2?c12FwB7MV1!877E@z87zxmu3GWJm z#6^6tSUY@VT}pg3E%Ro-3jL6Hp=q@eh6PtU7|F_;{bZKQ2%&EzM{mZNo8CxnO$^u_=~Z z*&x^iWBc)5Z2=RDvix}ob*N=O&=6aYh4sUDmP#%x4$f05xuQhfAZ%pmR$NiAG@?k9 zqlK{{mQ*N7*vML=X{VC8rYlO+8^Y$|5F2qGBTT0W8*RPmZ^7n@Hdfxq@?(S%Lp7;4 z(X@>~phTMutX#O+aN&k_(b(%C)+A2j;a#MXoF0Bk0t%8W+y#21ZK6leX!s+kB&X+L zL(fB?N4lVPB+F>w@rqcIxAU}V=P9%!YoMAm3ZjLPE0*N#)HdzZLOYu3QcaS5G`$Jh zT#~o*vT5ffv}fDO%+kEWz(*fO}kn` zJ9?f#HEC2v)61sKB{`oI0j%`M=SAbq?_y2Tcr?AlRA}d8LyzKzwVjX9jx*OJ{rAU1 zFR@)6w6oe7y=lI`MelXcu0OrtTCmzWu(aW_KXD9@{q)B^Y7v{S4K|8gR&2fkwg59W zUx(Pp#$s5Q85?^IOG$;2G!iLt##%;VU9^i8azT5JcXQ7+5iiRjP)Gh;6z+jx5Sds|CckBM2r+y zjyyh{nK-3Gf>AD`!W}V}fnab*IgIqJLLT$`a+z}P!2(7pmnrv7u#_n}$nOiejO@M} zG$_CC4(};%uZCAFS0|7%gOJwC*@G|oDJ`^uJQ!VZ5AS$d=Tg$$% zoDSe$!taO7v)nty!av5FO5#t-t;i~vFruW~s-8LjC{ZO^a;thQio`F(cdADeM)sro8iuU#;TcQtb-|XL0q%h6n|W}t;eqy*T6w_#VL?ga!39!Hz&2x% zTl%jcRvuicmAEO3wAM}3WN5K~Zp`NT z6=v``xEFJ4}Zq^(%5~z_tjRa~WP$PjF3DiiSMglbw zsF6U81ZpHuBZ2=N2~;-!r}?)_`N!qUL!0?6ScLh1@Xs6L0CQKsp2A$6=IxH23DAlN zJrkfOhxAN<)^HZ$q_v|JIBVf7!%6e_%{Xc9PtOQwrRg9}dM5A@PQv{y&gwi9kk9v} zYjIBQC(r|$O21ox5u!mWYU1ye1=g_pTK+?2R@+>S9vM&M?1l3{!VjzT5qTQKD3jQ=Ki#%PWv%w;!OK1Ih?I< zw#L~8X8_K&INRaG(!a2Okai8yUcrtygK>u7q+NeuI6L9&jFa}sMBw}ld+lgX5AD{W z-7-CJK8cg&-FOv|a38;ZW_@u+;*7#cv+Nk0u{f#yBb@a8|I3krx6|nHCLIntw&`6! z^ghnVIsBQsu*Z?)MRvrg=~H}ugRlNVLGeNIl|Unfx8qF2J8g)Mi4wTl-`GuRp-((i zx8b-fv8FqAlhIr4=!L7)_koat_zon3NN*Ub9BUxxt8kmfPj%_jo-|?b;L9C*Z|R*x U#Dn;brN`r^$M5KWZyEjn1G?QvdH?_b diff --git a/src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.xlsx b/src/test/resources/files/experiment_controller/feature1465PopulatedDatasetImport.xlsx deleted file mode 100644 index 882be4a57d084ef0981cfaee6b297d061ffb770e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12786 zcmeHt1ydbs*7d<5IKkcBA-G#`4i1o)!K^qGbA{54gmc1{{J2S#c!Z5Vce#N8KM6){u)WOLK$zJS?+C6?=wDU zel^S>t|-|Q`6tAH#%w&jv_hmnp``*vyoU#Gdqd=b!+sOGP_s5V@y`7)1g)KV`wOKu zh3pI7SQ46|#49A}YB2Pz$*Wnr7C2uVQ(k&;IrNi)zLH=G{S64N)zrHM>dkqsWMNhK zcQD-hS2o5w1b%rL8i*{o0zI+}48{tU&utW8dWuHK%UyG&=(LcQwBoX~YVt3jAr~Yl z(?aYrxh}kZl$z<>mAhrVl;W~yi$~stIzJ}k+3j_f8Z^2ow%1n0o>T$BKu?PPeoVp-mD+eaVU+4b?<$rM` z{^ilj;^Y*1nGu3dB_BfuZ)R6wP();0ge6-^RJ?tqz9ZL1=8@yAbW-7?sNe=by!UDM zejHw2;fp*NB)Q&XFAGOQ=Ot@!DGyG*w|9b}r358O*q3eep*qiA&0eRzmv*CeZjYud zZTgZcGqgq~F?}XhjWW)th70$$06!R$Kh;m8Urv41;I;y6Rz&%@Jh-ZnGv^?FJl$tL zsqhD)U?{ij(NrqNpo4+wVuk0R70J~duBx&bw|SL87Kn$`P1ne(>q0EO^X-!-gM7xI z5-B^{E$jIE0g9X}AB{Tpvu~MheVlN;rGr0*{KJX9tiIN%{|pl8;zIrV*EjMW0sz1T zz=FA2G5sS>Tx>y>2DY}AziQaOVg~HBkiDY(cOPwWqZWP42!W?T9RZW=_Hoa^VIB{f zp!SAk80KoU5IHNuVnMfuw1g9ms{G`MJaYC6mJyQ&?#oRAGN)A<>Kc5=mZkKu`Zqc^ z3TijRq4jJnH%J%}l2zr5OdR72mhuWltwN;5TuOSWy#Wa-n)vL-0y#3t{R6@dxQ28A z704~(>brND5uNXH(#$_)Dx$dT;`VtX4BgpUud!=KMt6AQO`PGt4~4Q#c%WuK&Xoe*yF2_7s)h3DnGJ~7_c^4_QcGhj=~S92=ElJX);kb)Y2kE zdmY?Y)3J{VUV}co9Kipb=R9dAFHyaEO#%!6fc)wbub%T)pUGF%wB6)F_trIk5#7F- zvG-*UON)X9LsV)l!7f~J!%gi&QHbZViic`(4;rY_vg%lqUb=DDdKuBkl}2EGHb8aPf8*8kbP=7N88c2tlEnb{q7n!y zKSVK1+%3yLke(^?kFSrRn5|00E3+6fD+KjBs_V_$^JyvRIE=`-Az9cWM4$=1sORLi{XOMQjY6RSXi#1Wz&s@o2+!n7iE-Iz+&j|^X77u$ZZ2M%r2FyoS#1^dn+F8h zm~EbLzm4r&n_jd>p?Y1#%jW_qE59;-TK>@{6ZOCfeLCl(Od$@Z??1ui!`K|b18t*^ zJYqid&(+afqPfoZ27KOwIvF$*As7fUsrj5HCsyFc5wiKjD9&>v#poL|=l?WqHPit5V3l#+p%m+*TGelP#M}?b*ZT zF2DcM&7PK~r6<|Hy76>B(C53xdbD_%Qvh}727=kS6Ke$0uX{P0wR5ZVzTD%+`=#IJ z#q!%zb`ZFV&LerU)`pzL$hH$-Lswi7e2&pcd9$?tC zOoX$xz0Q1A5gAj6TT-7-7vt#0&obX`tJjcTUP5+fz$1TB^fM-{By9xMXeg#vWULlx zL&alhP8M|0p$<2DxxKW$owkf~QeIFenTs_tvjNj*@0HiB;Z@|w34!PqsIwJI#2e66 zAHt?gjmG7S?;+}uFan)t87Etqmr9fBHrG2xT;Wsw;`M#SM1^w#w~(&I%dCG<*d4v% z#$U95RgqVVoHC*JIGbjSkV&O2ov~A3lu?y%ZD8mY^mJyx@?4YgyprDag2Nd+C4_ek z)$>GjaNPMCG5?v3d&GBnr62(S155w_@gFhe;OJ&$?C>iKfB&c%oyURhT{-jO`#5F? z$BYtG#t|P@H`P$Wu3TJ+l+veRNM>geUw*&l5kaC)880isdR)f@$8qh_;c&)%8zE5c zV8Rz$^Vxr3u(>)`*p*ai7>nip$=)P*@C*4ajfsCewzXuCK%~#JxyleC8M2H>J{U^d zr%9z@12c8i8glLm8!=(7wjr@pL%3d$zN)yCokp=7cdgbkQW?H-6nhV|f;`xe{hhUH zO4uMg{f`V*TF!Tvtmj6z#@*s!dk9}D2#~4zm*Auu6M+iHa6{%wpQMe>p@kS?joC>a zw8-dc>}o{87rZ5-Y!@p&c=8QHCMU**VkEPi1GHlay|*@bvy0W%-{z}DzMnPk+*q6&*uT{D$yhm*dmt3Ai5Sh&I11xBAw-(ev^P@XDk` zMbYLL^kEq6TQwu#Td?JE@l+8d3*>&l<(%>sDp_hE7xfQ=;^zYDbr;6`z*=6y0(2(G z0okvPt$D3)<8Zdiv4<$OGqFNoLD?Jy=4)J8tc5Ozi;Y>Q&^BIO^c1bAUO2$f4oV*! zB(pVIADkGDb>r*Otz#9uA76_Wg)8|L)!J`U1myIDQYZqeVK7j9jFvwt-#KUEcqWd6 z?x1Q20)vL5j>$3-b`=K_3BR}=el|ZsD#N?%c{jKCp{uzJpPFja1_BFeS@uAARwZ22 z3<7xLM4+2fd_VKnGFm;{30h$%DVS>JCX?y9T0e2~4ox#!kga8%d$J_rJ&@}P=!Gto z_Qf6EFC^lWf{}2jWp+WsbT4EGHRO@ZsH^@(j+fv~H1c{+DNuQFix#Y)L;6{_`Z>uQ zsh~7oc?6H84xQo&ELENYT=iLziiJFeBlOu@V7EOB-Q)fovd5I77Fln-lw8*dj;|A! zu!PmaETj$U%)@3ypyx7D&nua9maX?{`<$`zo1j5z5_b>CgRD`t6jTyi%4Xg z@D2hrcOg5!TX-D@J|8!>vbQ*D>Jh@C!bv01S#nKq2pCgK=w`S})I`=RJY3-){$LGV ztYLqJXh{dPP(a>GM@iJkWLW3%9F*^v7;v1f+Br!@Vi}5koYh%!*!h+k=cnRxy}u>* zBVK7-|Ni*HkTrXIcC(3@g2*%;%38$Z(9C6f>yRT)xYc~eL+SL}wj8pLH%ug7c~`3p zCOGNwjPcHqTbr?+pwqlpLcx}_Ph#;Fvb_d4Od*z8+5sUZ*a)5uySyjtKPhjXM^5c- zJg$kYzu7MlZZDmM#O($uuHKhi0bMXNAbZKAKbBLKX*;++a{g`3>Bd6okbkY={lvdG zj6dqQqnWX_G1DJsmS36YNPRRMj}yHU>s}DS!R7JYRy4)x`nXm63b|2c0)BnNk>Uq7 z*0?sjx8M}qS4!0R1<4||{0UP+5Qy!U(QxE-KZqu&MihMXU zvtg_@#_MF?5AiwW~qjkZi*vUCt~gF;f=5$Mer07a^VeK9~D@jv!MwefXYF18SzHIqlBR!USueK(A7#&+RZFG>6M^QT>w>`MGab69KGM6Bfr zqjVYIW{)jLZBuD6vdVZD@e^8DAy@a5a%Z(@1VL%5Xz0#7KkKG!RoLYTN@YOkG=(Tg zk3xyDPShauTaPhu;L`c%s&!@W7t2|GPa`t@*L5&^hZgVje=}oLJW}EoV|ho=5pJ~2 z3i9cGPg%i4F6RD;=HlLvK2T+4(ZoI_Tuuwl5#>UHYQJlEcimmJz(U0xXDB$pMLZ9v1Uh||Hsa_(%k#rf|;F#)F zh8~!4T#4lz4Ja9w&iLjm!Me)S*52P0lR6A_E&mQzY(-j|YbQg+Ju?bPZv7qj4$*e_ zG|fEJK66}ar_^C2th#BkJLZX}wjG_fZS8=b?kfLr2PBTC1rG6bg3SvCL8ptCNKEg>(*N2y}@L?Qi3^BfpM^)lfaU3=_J5IY2{2Yo0wM@T5~4 z?}_=wn%OB^c$u; zJ>Sn#Xsg~~BS|><2KBX7#KjA;Z88Fe`OC6m3stiP*cj<;+Pf6`?+IiKTlSQKRIE$U z69^ij#w%SJpJcC8T-hjd4D&;r=rFP;${*w}T%#kWr?K^AP4E<*uoZQYM9_6i19cWtKTbM+oCF>u=W0mU@4R8Q z#=vmhxn37<-w6?35)L-OPuMfWD5^_si>P=rmMY;M&sLlY1~gz|BaU`dUTm>T^zqu< z)my+e(vq=T-o4$&?wxmqZYG=2#QLn<Lh_DqF)4FpH8TnF5Ag#{UT6-9{`u z7oDl0EwFi#X8=`LGBzWj&ALgqZOnLuLv%AUWRI}Qskh1ZeyXiiT&`oIhec4B;kdPS zlHlS$tDq7b%pB2h5~4qp&b+-J{@ylg0^{Px=Wgf2Riv~J6tPVvqrRWQ8%*HP3;!Ok z9tpeF#!bcW;!R@(vm(8~kbtY-iJ}sql#AL>8sv=q;oDP{)`1f(^15EEq88Hh;XRm^ z{)D)*dF8-71JfUuKWcD9oSg0Q zFRBXSnh0UVQeA$xKsdv|3|PjCAlm3;Dqd(v61O8I9e*?*9d}G`7EFqzOtjmZ%v9gR zYnIsN>oiqIp>HmMW)@Uz)JiinPK3$)8#NL^D^eR?^qTDO~ zKL1U^t5ykTn|yP7M0tnS6(`P~eI!)N;(S2~YN@&uK^qOjI}QCc7>*qSqk&N<9U1*d zwt5BB_a3>){A*L1pYnG;$1Mm~@mB+rx8J7w4B_N{6KCI7?^(?(j~DFUKlO)Uh}cMw zf2i!%f0q$lTo$<)d^u1;j)Mux5rXtVi`sxX5#BhZOWHE>8CcSf4Ej-ge&?=ohC+2W zN@;nrzWPPC<{U9eoB?CTv*>Y=gqJ`d^~p3Qy??6`3K!cmLu~+q%MU^<$h?dpqDSOb zt_}ei^=U5z`^GENGtL`U;S8Ig>$AaL+hpFSoR@^i%&%4vq8#Z3Who}&-tP=qnVFxJ zfo%rF$A~!7V1D*Oo+DF>95W<~|K@(;Nx_@+Hf1ENt9)p&c zQ9LpFkx^o-!Ht;NoJc&|phQy(1<`m?3Xhj>jBqp?>6V*1AhCQ7!g`J-`7bCx0>e8g>C;hqVs5xp6t>g&||YklmZ4hvQR{@XF2it zN2jEZqH`GJXT)K_R`}#${j+10*b4@FA%6VONIH7-{&OY6wE`svDq=1m!IBcWEOCcu z%Qv4{4c?^j1ftg^UE+`%_aUG=!73;sSg};~b`e6rFy){pP6?FkdoEpQENQ7HxNq+f z6O#djB?q=*jzrJWi{e0p6nbvaD(r|Z1IPyR7Om5gp>x9mecFeJr!kcLudw+T3xx%1G76h`2~#eY?>u*m9#M%5 z2Vj^b$pONYxtSVitM@q@Od8E^r2Vytx==7oRXxQ+--vD|^>dopS*aGbJ5Latl+Z5R z8=J@_f3d^;jJ8l*@PNn@KJew|4?@G26-N%_W*DHEt`x)?6E_o`A=d*4R* zRk#pUDXNq;m$$fWJ7o^cgNZ0vDMBK-x1uawGGxsJn1ffvg4gwNm*9@>3w!#CZ|pZ& zJ#rcHobi_&_H#rIMws_STU6v>e7+?7sFf_1%`082hMbBS&Y%DrDU{ArasHMtjJ|Z% z!yke~o3DCp9aq;S+Q00sVYf;%Djo6zshM8qAr_d5?~!{-yu;~ zQEuhGrU!;_MO2iV`gWnX+swHmsASu3y$4lt%sn^2{~1l61{Mv*i(KnT?x6&wD8cbaPmIGWNTOB zi0IFVs-cmTRk)I)mmVLX`!FaaM^~-NI>Dxpq&AvQGKzLRy#SlZ6{`h zmTuwT#o;d4$>+*SN|Yd_YJ->^Id}A4!Bp_s?!b$<&GK=P8a|gVRo*MkKKHRN$;!^u zTss%Mqvf!eiJ)}GbR>7MXdomcx+rMML@caa)^|;`sW{@fbo1hVxH9^Et#YMlk-oFd zMG5mIIOam8eD9GXE$2ZxQNpxEEZgWj#f@#pPABa#*RgJ=UN)CDbl>;Rs|;<$-C?Xt z4Gf5YaT^)u7QMGB@>uu@V-L_e%c~eeQ(;W1{zoK;Su3C3*Shr^t@|Ctfj@Vw-fA<0 z6xx*B3c!mJs72u^t>fCH&$QETMfA8$*I5`!CfuGR_PE#wRT7(3e*|`9WbQFB-E#`o5XI{}Kiml33vYtgn7h=uas)Zx-6)cVn|3nNt;$?(o)W_XsUEkuUU9$b-Chr3 zjIdG&na4y>vw8-oEn7@W(@lw+Yo|EkOM=}z3P`V~6c|$nupQ1BiVVbv>k|~9QWCBL zQ}h)Z)HHEaAp<0ZPpDeLN=M+O?)g)pnXv%kYrnsKKc z`XUkap;vzPX`u1+l@05x{Mk!E8%b>ganM47Dg}h0_b}d(KAcQ*mC#LRv6sAl*cj`Z z_Q>m1TPQXTiL!Tuq`f=(ko&Zljl%OT=d=roG6GKa32u$GJdFUBpXPeRcc-x#4 z$`=SAav8r1*3N*X6n^enyu+4kSe2Ql2c-iz3FRPtDN{k_@wKP#pV?}cY#CnOe^D~z}3)$K!hb84G?3;P=dDrO+H^sl${>&{((a44yi3)S! z>uK!UHIuYf>e#uox)iHumVqxi%XLUR44MWx6~D=>Jp6VxYtD{i`qQo@rBIz`>=cCH zP0O4Z^&GOywcRZ6J~z_o+{PgZr%C^lD|&y6R9F67Uvy}syoloQeT&^tTH>M5C8gm7 z#;3Q36qn-U%Y%~yts+yrk0C#KmB1u(1aaq|vMeiYacwNGN$z2?1WCs>`c#mbK|rX2 zazhIVCm0{(na&<|cKju#5|LC0MN>ub;ONTf`gJ>_e$d}{(vrdS z5Y}XTD}I0mS)C7guwx0Pp7A14cKXOHba#&j$0-_VBuEm6q3j#oLMt-MhX-+3(~07!u+&Uj7&dTNZ1* zfs!BSL&{6~UWevfTRrE~K}VIm4|Krht;Zn*)K#MbcE84gqyb+?jsrSa>_(qr5;C+y zyWJ(Cj~n!9K~$#;k)HOd828`{#l3`x zk4*mQM9^_c_zX*~V9<{(5%L~!FqV3qRgAt!SImCjIPe)IV4Di-r!(qW8SI)EJiW3n z*~{fHJnnC^Y+Zsfc|5e?Zz@ndYZ9XIIzqLmYU)s%VQKB z7)`?S?1aJQK4k9;^91@fjZx*F@G6h7weVJkY8D{Z$^yBjI9hL1X7R|8e!kx)_!4J1 zi92esQWpNU#ljX$IjE>fS9qqVZN{WEl|x3WR(Xij?Iw<9TKDtNroo!-2hM>Q3d^Rj zv9(N3k}&PYdZ$2@pe?Xt=b@|R4<1q>ipP*k5 z-H9v7V;NJM&xgh0@mc(+ z7s4t}=N8p;0zSv@d3ge>cVOhe_2D~ur}XIa>saqUt6tbex-9+caAFqW|J26b7(=YC+M3~jc=W2w418bgi|8ye*1M-Q zp5?3rNtA=ACMO)nObNy=o7kp~vAy?j*c%AhKUJ77@(viokA2Sm@@U?m>TYcTz#|ql ze50hfysK#R5iu?eSU<5Ym^c-u;W<+}Schx&Ls&`79N#oHzm0@IwBmFFsQMMBjk+Or zVj)T_AP4g)J+0A1FaC&-N9xLbT7a2CGyfx~`*<}zPN?S5dX`!)8GO85rauAk`}2ms z4yZ6uF%dDA@gPDc%MDkK%qVSKI*Q7!8V0w--&5}VYpB>o>i8a3NICZ+OcJAyFTf8KScX{M_d zt<%liGNG@VI`=c4K9NX>o=M@!<|@q&OB1WIjg>cbxo}E~9VKPH4ME z6R)SY+qg3lk&+@QIVtN}H=oa+{t)W?(NiTPp2`O>qkq*6`UwCpXc81gqhejuJ6QE4 z9_p~d{-nIN3UA4Ca<$Z_VEA2EUcp0Nv1 zZ~5U2QHgD|`Z7VOcOcTww_uMLgC#m0UPX~5>2KREZ3EN$u5z3NL&5kR2(YPLz2uFK zGUFFY-_=aPmKnuxOAxZxq!0`HrY6M3itWWthgHe_Bo8PkjyRG zA)K}CU~e|Bzy6F;Vq90Xu}LV^1N$!622jSCWHIAhy=Qrz9U0<&=o|`(%6XCMk4*$9 zrJDqp&RA(Ws6il%M)r?Q)ue71DS}{edL=>YmhuBs;+xo@5Kt6J7N?nL57xXWZsoyT z!$uDWeKl)C=e(%Y7%J4;XQ~viXEtoMI>=t;PX7%@GiH2IjVuQQL}=?!xc-0M4JU!j z)IEW>rQHcH_-8xJ7sZ0qBZL6iX>bPy>4zf6h{%pEDcQD8EI&q?p%4YueaI3zacsrX z>Le96bn6M+Ctt2WY!a$6&Gl91h&#1I8e7N*{h#;%vh@$k-LVWo&jeNzgT!qU=i)cx|P!t=!j${K8L1KCv5F@KUu zc<2^<=^ZahJ=7i{(t@NQJi^XocUw5)AHw=82K*8uWF@*$Y+x=i#%DWz`m zMM7-?D0I8BYGKi=@YHoI#$3zCbO_XD1CF$Nj6 z@gNX5)EDpfU`q7IJ}@IxMKhr72aXSp-z8VJdVJVbXH<$UAYBU+uhzm`+_ZOH9eqSu zUNDoFc&X%4s5PeTV;m2^`;3xZfj$my{9Q$=sj-6SxL4&X1qXVAnM_<%(Z}wN$rhcf z^MC9}e-|aH`Pe^Kyt=sktBa%j)y4Jg?EdHBude;iBP0H=bSGZ(A`rpv=(7y#WtCoc zoU^HXpV!@o(Fx@@RSMIZs{Ha&1pDuHZA47v!BlzmxL{MBj z;>SI1U}9M`re$7{p#+zUS$5u*auzPX+~}A{qzpGHQ!XX@eZAfZ>iv~ar-hh|trm)?yGoSoFCs>)8qmR;+!Xj7zV^<)*rNY0>iOOD_i6l}rhO=XnEpPS{~h4>dBUFnqOaqHe=ht#rVGEL z{LXOwiE=~mk01qyN*Bh4!@)Pdq?mm1^}Qy0|5L-fAG8czh~RO gnzPdX#r&VS7bpYu>Lb6z;_!g}SJ5~o)31O35Au7#r2qf` diff --git a/src/test/resources/files/experiment_controller/germplasm_import.csv b/src/test/resources/files/experiment_controller/germplasm_import.csv deleted file mode 100644 index a5b424b86..000000000 --- a/src/test/resources/files/experiment_controller/germplasm_import.csv +++ /dev/null @@ -1,2 +0,0 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -,Germplasm 1,ANE,Wild,,,1,,,, \ No newline at end of file diff --git a/src/test/resources/sql/ExperimentControllerIntegrationTest.sql b/src/test/resources/sql/ExperimentControllerIntegrationTest.sql deleted file mode 100644 index 749796fb8..000000000 --- a/src/test/resources/sql/ExperimentControllerIntegrationTest.sql +++ /dev/null @@ -1,37 +0,0 @@ --- name: CopyrightNotice -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - --- name: InsertProgramGermplasm --- insert into program (species_id, name, created_by, updated_by) --- select species.id, 'Test Program', bi_user.id, bi_user.id from species --- join bi_user on bi_user.name = 'system' limit 1 - --- name: InsertProgramObservationLevel -insert into program_observation_level (program_id, name, created_by, updated_by) -select program.id, 'plant', bi_user.id, bi_user.id from program -join bi_user on bi_user.name = 'system' and program.name = 'Test Program' limit 1 - --- name: InsertProgramOntology -insert into program_ontology (program_id, created_by, updated_by) -select program.id, bi_user.id, bi_user.id from program -join bi_user on bi_user.name = 'system' and program.name = 'Test Program' limit 1 - --- name: InsertProgramObservationVariable --- insert into program_ontology (program_id, created_by, updated_by) --- select program.id, bi_user.id, bi_user.id from program --- join bi_user on bi_user.name = 'system' and program.name = 'Test Program' limit 1 \ No newline at end of file From 17e3137c5a2f317f269f4a3519c2845d8ba76223 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 7 Jun 2023 18:41:39 -0400 Subject: [PATCH 25/27] remove npe --- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 3 ++- .../brapi/v2/ExperimentControllerIntegrationTest.java | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index c95cac171..845fa4d63 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -257,7 +257,8 @@ private void addObsVarDataToRow( } if (includeTimestamp) { - row.put(String.format("TS:%s",varName), obs.getObservationTimeStamp().toString()); + String stamp = obs.getObservationTimeStamp() == null ? "" : obs.getObservationTimeStamp().toString(); + row.put(String.format("TS:%s",varName), stamp); } } diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index 2dc8ed364..fb5f79f92 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -161,7 +161,6 @@ void setup() throws Exception { Float val1 = random.nextFloat(); row1.put(trait.getObservationVariableName(), val1); - // row1.put("TS:" + trait.getObservationVariableName(), "2023-06-07T20:47:23-0400"); } rows.add(row1); From bddd2c5422bf9d776ca32010aca6fb8cf23672ee Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 8 Jun 2023 10:41:30 -0400 Subject: [PATCH 26/27] improve code --- .../breedinginsight/brapi/v2/ExperimentController.java | 1 - .../brapi/v2/services/BrAPITrialService.java | 9 ++++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index 826061c88..4e4a67f18 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -105,7 +105,6 @@ public HttpResponse datasetExport( return response; } catch (Exception e) { log.info(e.getMessage(), e); - e.printStackTrace(); HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, downloadErrorMessage).contentType(MediaType.TEXT_PLAIN).body(downloadErrorMessage); return response; } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 845fa4d63..d92a8f739 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -124,8 +124,9 @@ public DownloadFile exportObservations( Map ouByOUDbId = new HashMap<>(); try { for (BrAPIStudy study: expStudies) { - ous.addAll(ouDAO.getObservationUnitsForStudyDbId(study.getStudyDbId(), program)); - ous.forEach(ou -> ouByOUDbId.put(ou.getObservationUnitDbId(), ou)); + List studyOUs = ouDAO.getObservationUnitsForStudyDbId(study.getStudyDbId(), program); + studyOUs.forEach(ou -> ouByOUDbId.put(ou.getObservationUnitDbId(), ou)); + ous.addAll(studyOUs); } } catch (ApiException err) { log.error("Error fetching observation units for a study by its DbId" + @@ -247,9 +248,7 @@ private void addObsVarDataToRow( BrAPIObservationVariable var, Program program) { String varName = Utilities.removeProgramKey(obs.getObservationVariableName(), program.getKey()); - if (var.getScale().getDataType().equals(BrAPITraitDataType.ORDINAL)) { - row.put(varName, Integer.parseInt(obs.getValue())); - } else if (var.getScale().getDataType().equals(BrAPITraitDataType.NUMERICAL) || + if (var.getScale().getDataType().equals(BrAPITraitDataType.NUMERICAL) || var.getScale().getDataType().equals(BrAPITraitDataType.DURATION)) { row.put(varName, Double.parseDouble(obs.getValue())); } else { From bf3e994d0a45a4e39d4d3f2b767cf14fafb0fee4 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 8 Jun 2023 11:32:57 -0400 Subject: [PATCH 27/27] add comment --- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index d92a8f739..4faa89dba 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -158,6 +158,8 @@ public DownloadFile exportObservations( if (!requestedEnvIds.isEmpty()) { dataset = filterDatasetByEnvironment(dataset, requestedEnvIds, studyByDbId); } + + // update rowByOUId addBrAPIObsToRecords( dataset, experiment,