From a5b8a96a08fa1c0294463eea3d769b5ca19379a7 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 4 Mar 2024 16:50:08 -0500 Subject: [PATCH 01/44] make serverinfo calls path relative and increase read timeout --- .../org/breedinginsight/brapi/v2/BrAPIV2Controller.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java index d6e4dde43..b5249516e 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java @@ -43,6 +43,7 @@ import java.io.IOException; import java.util.List; import java.util.UUID; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Slf4j @@ -147,7 +148,7 @@ public BrAPIServerInfoResponse programServerinfo(@PathVariable("programId") UUID .build(); for(BrAPIService service : programServices) { - service.setService(programBrAPIBase + service.getService()); + service.setService(service.getService()); } BrAPIServerInfo programServerInfo = new BrAPIServerInfo(); @@ -230,7 +231,10 @@ private HttpResponse executeRequest(String path, UUID programId, HttpReq } private HttpResponse makeCall(Request brapiRequest) { - OkHttpClient client = new OkHttpClient(); + // TODO: use config parameter for timeout + OkHttpClient client = new OkHttpClient.Builder() + .readTimeout(5, TimeUnit.MINUTES) + .build(); try (Response brapiResponse = client.newCall(brapiRequest).execute()) { if(brapiResponse.isSuccessful()) { try(ResponseBody body = brapiResponse.body()) { From fa0c8db48f12c79cb24e09297904c49694e3baed Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 5 Mar 2024 00:39:31 -0500 Subject: [PATCH 02/44] Added pedigree controller placeholder --- .../brapi/v2/BrAPIPedigreeController.java | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java new file mode 100644 index 000000000..59c1c7615 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java @@ -0,0 +1,123 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.annotation.*; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.BrAPIIndexPagination; +import org.brapi.v2.model.BrAPIMetadata; +import org.brapi.v2.model.germ.BrAPIPedigreeNode; +import org.brapi.v2.model.germ.response.BrAPIPedigreeListResponse; +import org.brapi.v2.model.germ.response.BrAPIPedigreeListResponseResult; +import org.breedinginsight.api.auth.ProgramSecured; +import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; +import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.ProgramService; + +import javax.annotation.Nullable; +import javax.inject.Inject; +import java.util.*; + +@Slf4j +@Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) +@Secured(SecurityRule.IS_AUTHENTICATED) +public class BrAPIPedigreeController { + private final String referenceSource; + + //private final BrAPIObservationUnitDAO observationUnitDAO; + + private final ProgramService programService; + + @Inject + public BrAPIPedigreeController(@Property(name = "brapi.server.reference-source") String referenceSource, ProgramService programService) { + this.referenceSource = referenceSource; + //this.observationUnitDAO = observationUnitDAO; + this.programService = programService; + } + + @Get("/pedigree") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse pedigreeGet(@PathVariable("programId") UUID programId, + @Nullable @QueryValue("accessionNumber") String accessionNumber, + @Nullable @QueryValue("collection") String collection, + @Nullable @QueryValue("familyCode") String familyCode, + @Nullable @QueryValue("binomialName") String binomialName, + @Nullable @QueryValue("genus") Boolean genus, + @Nullable @QueryValue("species") String species, + @Nullable @QueryValue("synonym") String synonym, + @Nullable @QueryValue("includeParents") Boolean includeParents, + @Nullable @QueryValue("includeSiblings") Boolean includeSiblings, + @Nullable @QueryValue("includeProgeny") Boolean includeProgeny, + @Nullable @QueryValue("includeFullTree") Boolean includeFullTree, + @Nullable @QueryValue("pedigreeDepth") Integer pedigreeDepth, + @Nullable @QueryValue("progenyDepth") Integer progenyDepth, + @Nullable @QueryValue("commonCropName") String commonCropName, + @Nullable @QueryValue("programDbId") String programDbId, + @Nullable @QueryValue("trialDbId") String trialDbId, + @Nullable @QueryValue("studyDbId") String studyDbId, + @Nullable @QueryValue("germplasmDbId") String germplasmDbId, + @Nullable @QueryValue("germplasmName") String germplasmName, + @Nullable @QueryValue("germplasmPUI") String germplasmPUI, + @Nullable @QueryValue("externalReferenceId") String externalReferenceId, + @Nullable @QueryValue("externalReferenceSource") String externalReferenceSource, + @Nullable @QueryValue("page") Integer page, + @Nullable @QueryValue("pageSize") Integer pageSize) { + + log.debug("pedigreeGet: fetching pedigree by filters"); + + Optional program = programService.getById(programId); + if(program.isEmpty()) { + log.warn("Program id: " + programId + " not found"); + return HttpResponse.notFound(); + } + + // TODO: implement this + List pedigree = new ArrayList<>(); + + return HttpResponse.ok( + new BrAPIPedigreeListResponse() + .metadata(new BrAPIMetadata().pagination(new BrAPIIndexPagination().currentPage(0) + .totalPages(1) + .pageSize(pedigree.size()) + .totalCount(pedigree.size()))) + .result(new BrAPIPedigreeListResponseResult().data(pedigree)) + ); + } + + @Post("/pedigree") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse pedigreePost(@PathVariable("programId") UUID programId, @Body List body) { + //DO NOT IMPLEMENT - Users are only able to create pedigree via the DeltaBreed UI + return HttpResponse.notFound(); + } + + @Put("/pedigree") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse pedigreePut(@PathVariable("programId") UUID programId, @Body Map body) { + //DO NOT IMPLEMENT - Users aren't yet able to update observation units + return HttpResponse.notFound(); + } + + // TODO: search and retrieve endpoints + +} From 89120b1ef6fc52098123d9b55c9ce5ea06442e85 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 8 Mar 2024 10:48:10 -0500 Subject: [PATCH 03/44] Adding pedigree dao in progress --- .../brapi/v2/dao/BrAPIPedigreeDAO | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO new file mode 100644 index 000000000..20857aa8b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO @@ -0,0 +1,81 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2.dao; + +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.modules.germplasm.PedigreeApi; +import org.brapi.v2.model.germ.BrAPIPedigreeNode; +import org.brapi.v2.model.germ.request.BrAPIPedigreeSearchRequest; +import org.breedinginsight.brapps.importer.daos.ImportDAO; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.utilities.BrAPIDAOUtil; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; + +@Singleton +public class BrAPIPedigreeDAO { + + private ProgramDAO programDAO; + private final BrAPIDAOUtil brAPIDAOUtil; + private final BrAPIEndpointProvider brAPIEndpointProvider; + + @Inject + public BrAPIPedigreeDAO(ProgramDAO programDAO, BrAPIDAOUtil brAPIDAOUtil, BrAPIEndpointProvider brAPIEndpointProvider) { + this.programDAO = programDAO; + this.brAPIDAOUtil = brAPIDAOUtil; + this.brAPIEndpointProvider = brAPIEndpointProvider; + } + + public List getPedigree(Program program, + Optional observationUnitId, + Optional observationUnitName, + Optional locationDbId, + Optional seasonDbId, + Optional includeObservations, + Optional observationUnitLevelName, + Optional observationUnitLevelOrder, + Optional observationUnitLevelCode, + Optional observationUnitLevelRelationshipName, + Optional observationUnitLevelRelationshipOrder, + Optional observationUnitLevelRelationshipCode, + Optional observationUnitLevelRelationshipDbId, + Optional commonCropName, + Optional experimentId, + Optional environmentId, + Optional germplasmId +// , Integer page, +// Integer pageSize + ) throws ApiException { + BrAPIPedigreeSearchRequest pedigreeSearchRequest = new BrAPIPedigreeSearchRequest(); + pedigreeSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + // TODO: add pagination support + // .page(page) + // .pageSize(pageSize); + // TODO: other parameters + + PedigreeApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), PedigreeApi.class); + + return brAPIDAOUtil.search(api::searchPedigreePost, + api::searchPedigreeSearchResultsDbIdGet, + pedigreeSearchRequest); + } +} \ No newline at end of file From 30309e2332299a1c4299c3c3426ec33fba812319 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 11 Mar 2024 10:14:04 -0400 Subject: [PATCH 04/44] Some more work on pedigree dao --- .../brapi/v2/BrAPIPedigreeController.java | 35 ++++++++++------- ...BrAPIPedigreeDAO => BrAPIPedigreeDAO.java} | 38 ++++++++----------- 2 files changed, 37 insertions(+), 36 deletions(-) rename src/main/java/org/breedinginsight/brapi/v2/dao/{BrAPIPedigreeDAO => BrAPIPedigreeDAO.java} (54%) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java index 59c1c7615..04ee62d41 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java @@ -19,10 +19,12 @@ import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; import io.micronaut.http.annotation.*; import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIIndexPagination; import org.brapi.v2.model.BrAPIMetadata; import org.brapi.v2.model.germ.BrAPIPedigreeNode; @@ -31,8 +33,10 @@ import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import org.breedinginsight.brapi.v2.dao.BrAPIPedigreeDAO; import org.breedinginsight.model.Program; import org.breedinginsight.services.ProgramService; +import org.breedinginsight.utilities.Utilities; import javax.annotation.Nullable; import javax.inject.Inject; @@ -44,14 +48,14 @@ public class BrAPIPedigreeController { private final String referenceSource; - //private final BrAPIObservationUnitDAO observationUnitDAO; + private final BrAPIPedigreeDAO pedigreeDAO; private final ProgramService programService; @Inject - public BrAPIPedigreeController(@Property(name = "brapi.server.reference-source") String referenceSource, ProgramService programService) { + public BrAPIPedigreeController(@Property(name = "brapi.server.reference-source") String referenceSource, BrAPIPedigreeDAO pedigreeDAO, ProgramService programService) { this.referenceSource = referenceSource; - //this.observationUnitDAO = observationUnitDAO; + this.pedigreeDAO = pedigreeDAO; this.programService = programService; } @@ -92,16 +96,21 @@ public HttpResponse pedigreeGet(@PathVariable("progra } // TODO: implement this - List pedigree = new ArrayList<>(); - - return HttpResponse.ok( - new BrAPIPedigreeListResponse() - .metadata(new BrAPIMetadata().pagination(new BrAPIIndexPagination().currentPage(0) - .totalPages(1) - .pageSize(pedigree.size()) - .totalCount(pedigree.size()))) - .result(new BrAPIPedigreeListResponseResult().data(pedigree)) - ); + try { + List pedigree = pedigreeDAO.getPedigree(program.get()); + + return HttpResponse.ok( + new BrAPIPedigreeListResponse() + .metadata(new BrAPIMetadata().pagination(new BrAPIIndexPagination().currentPage(0) + .totalPages(1) + .pageSize(pedigree.size()) + .totalCount(pedigree.size()))) + .result(new BrAPIPedigreeListResponseResult().data(pedigree)) + ); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "error fetching variables"); + } } @Post("/pedigree") diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java similarity index 54% rename from src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO rename to src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java index 20857aa8b..210d10bc0 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java @@ -17,20 +17,28 @@ package org.breedinginsight.brapi.v2.dao; +import org.apache.commons.lang3.tuple.Pair; +import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.germplasm.PedigreeApi; +import org.brapi.client.v2.modules.phenotype.ObservationsApi; +import org.brapi.v2.model.BrAPIAcceptedSearchResponse; import org.brapi.v2.model.germ.BrAPIPedigreeNode; import org.brapi.v2.model.germ.request.BrAPIPedigreeSearchRequest; -import org.breedinginsight.brapps.importer.daos.ImportDAO; +import org.brapi.v2.model.germ.response.BrAPIPedigreeListResponse; +import org.brapi.v2.model.pheno.response.BrAPIObservationListResponse; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.Program; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.jetbrains.annotations.NotNull; import javax.inject.Inject; import javax.inject.Singleton; import java.util.*; +import static org.brapi.v2.model.BrAPIWSMIMEDataTypes.APPLICATION_JSON; + @Singleton public class BrAPIPedigreeDAO { @@ -45,26 +53,7 @@ public BrAPIPedigreeDAO(ProgramDAO programDAO, BrAPIDAOUtil brAPIDAOUtil, BrAPIE this.brAPIEndpointProvider = brAPIEndpointProvider; } - public List getPedigree(Program program, - Optional observationUnitId, - Optional observationUnitName, - Optional locationDbId, - Optional seasonDbId, - Optional includeObservations, - Optional observationUnitLevelName, - Optional observationUnitLevelOrder, - Optional observationUnitLevelCode, - Optional observationUnitLevelRelationshipName, - Optional observationUnitLevelRelationshipOrder, - Optional observationUnitLevelRelationshipCode, - Optional observationUnitLevelRelationshipDbId, - Optional commonCropName, - Optional experimentId, - Optional environmentId, - Optional germplasmId -// , Integer page, -// Integer pageSize - ) throws ApiException { + public List getPedigree(Program program) throws ApiException { BrAPIPedigreeSearchRequest pedigreeSearchRequest = new BrAPIPedigreeSearchRequest(); pedigreeSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); // TODO: add pagination support @@ -74,8 +63,11 @@ public List getPedigree(Program program, PedigreeApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), PedigreeApi.class); - return brAPIDAOUtil.search(api::searchPedigreePost, + return brAPIDAOUtil.search( + api::searchPedigreePost, api::searchPedigreeSearchResultsDbIdGet, - pedigreeSearchRequest); + pedigreeSearchRequest + ); } + } \ No newline at end of file From 3429f23083daae130f5c37b7e4758e252ca90bc9 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:15:43 -0400 Subject: [PATCH 05/44] [BI-2065] - Experiment overwrite does not do all data validations on phenotype data --- .../services/processors/ExperimentProcessor.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index b67305c79..c349c57d5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -765,6 +765,15 @@ private void validateObservations(ValidationErrors validationErrors, // had been saved prior to import } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { + // different data means validations still need to happen + // TODO move into separate method if not too messy + validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); + + //Timestamp validation + if(timeStampColByPheno.containsKey(phenoCol.name())) { + Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); + validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); + // add a change log entry when updating the value of an observation if (commit) { BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); From 95fda13b0e1b0f1913305291203d5b8af355e510 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:38:20 -0400 Subject: [PATCH 06/44] [BI-2065] - Missing bracket --- .../brapps/importer/services/processors/ExperimentProcessor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index c349c57d5..6ab1c48b7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -773,6 +773,7 @@ private void validateObservations(ValidationErrors validationErrors, if(timeStampColByPheno.containsKey(phenoCol.name())) { Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); + } // add a change log entry when updating the value of an observation if (commit) { From d7d8dc6670632d6063d9bf515fb16b8f1142879e Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Tue, 19 Mar 2024 21:13:55 -0400 Subject: [PATCH 07/44] [BI-2065] - Clarify todo --- .../importer/services/processors/ExperimentProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 6ab1c48b7..c42b843a8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -766,7 +766,7 @@ private void validateObservations(ValidationErrors validationErrors, } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { // different data means validations still need to happen - // TODO move into separate method if not too messy + // TODO consider moving these two calls into a separate method since called twice together validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); //Timestamp validation From bd77c55d7fd17db61827b223032ff306114fb8d1 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Tue, 2 Apr 2024 14:31:25 -0400 Subject: [PATCH 08/44] Timestamp Error Handling --- .../services/processors/ExperimentProcessor.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index c42b843a8..ec7588fa4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -776,7 +776,8 @@ private void validateObservations(ValidationErrors validationErrors, } // add a change log entry when updating the value of an observation - if (commit) { + // only will update and thereby need change log entry if no error + if (commit && (!validationErrors.hasErrors())) { BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); String timestamp = formatter.format(OffsetDateTime.now()); @@ -1061,7 +1062,14 @@ boolean isTimestampMatched(String observationHash, String timeStamp) { if (priorStamp == null) { return timeStamp == null; } - return priorStamp.isEqual(OffsetDateTime.parse(timeStamp)); + boolean isMatched = false; + try { + isMatched = priorStamp.isEqual(OffsetDateTime.parse(timeStamp)); + } catch(DateTimeParseException e){ + //if timestamp is invalid DateTime not equal to validated priorStamp + log.error(e.getMessage(), e); + } + return isMatched; } boolean isValueMatched(String observationHash, String value) { From d1edd9942d87e8eab73e626ad23102731248f3bc Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 16 Apr 2024 12:02:16 -0400 Subject: [PATCH 09/44] Temporary change to build --- .../org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java index 210d10bc0..3bcbe1e5c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java @@ -63,11 +63,14 @@ public List getPedigree(Program program) throws ApiException PedigreeApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), PedigreeApi.class); + /* return brAPIDAOUtil.search( api::searchPedigreePost, api::searchPedigreeSearchResultsDbIdGet, pedigreeSearchRequest ); + */ + return new ArrayList<>(); } } \ No newline at end of file From ab4070f5cf6b92aaf76d22bef5bec656eb342aed Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 17 Apr 2024 17:25:30 -0400 Subject: [PATCH 10/44] Brapi pedigree endpoint now filtered by program --- .../brapi/v2/BrAPIPedigreeController.java | 3 +- .../brapi/v2/dao/BrAPIPedigreeDAO.java | 43 ++++++++++++------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java index 04ee62d41..1c0b1b5b0 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java @@ -95,7 +95,6 @@ public HttpResponse pedigreeGet(@PathVariable("progra return HttpResponse.notFound(); } - // TODO: implement this try { List pedigree = pedigreeDAO.getPedigree(program.get()); @@ -109,7 +108,7 @@ public HttpResponse pedigreeGet(@PathVariable("progra ); } catch (ApiException e) { log.error(Utilities.generateApiExceptionLogMessage(e), e); - return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "error fetching variables"); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "error fetching pedigree"); } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java index 3bcbe1e5c..4fa59d805 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java @@ -17,27 +17,22 @@ package org.breedinginsight.brapi.v2.dao; -import org.apache.commons.lang3.tuple.Pair; -import org.brapi.client.v2.ApiResponse; +import io.micronaut.context.annotation.Property; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.germplasm.PedigreeApi; -import org.brapi.client.v2.modules.phenotype.ObservationsApi; -import org.brapi.v2.model.BrAPIAcceptedSearchResponse; import org.brapi.v2.model.germ.BrAPIPedigreeNode; import org.brapi.v2.model.germ.request.BrAPIPedigreeSearchRequest; -import org.brapi.v2.model.germ.response.BrAPIPedigreeListResponse; -import org.brapi.v2.model.pheno.response.BrAPIObservationListResponse; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.Program; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.utilities.BrAPIDAOUtil; -import org.jetbrains.annotations.NotNull; +import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; import java.util.*; - -import static org.brapi.v2.model.BrAPIWSMIMEDataTypes.APPLICATION_JSON; +import java.util.stream.Collectors; @Singleton public class BrAPIPedigreeDAO { @@ -45,17 +40,30 @@ public class BrAPIPedigreeDAO { private ProgramDAO programDAO; private final BrAPIDAOUtil brAPIDAOUtil; private final BrAPIEndpointProvider brAPIEndpointProvider; + private final String referenceSource; @Inject - public BrAPIPedigreeDAO(ProgramDAO programDAO, BrAPIDAOUtil brAPIDAOUtil, BrAPIEndpointProvider brAPIEndpointProvider) { + public BrAPIPedigreeDAO(ProgramDAO programDAO, BrAPIDAOUtil brAPIDAOUtil, + BrAPIEndpointProvider brAPIEndpointProvider, + @Property(name = "brapi.server.reference-source") String referenceSource) { this.programDAO = programDAO; this.brAPIDAOUtil = brAPIDAOUtil; this.brAPIEndpointProvider = brAPIEndpointProvider; + this.referenceSource = referenceSource; } public List getPedigree(Program program) throws ApiException { + // TODO: maybe use get instead of search + BrAPIPedigreeSearchRequest pedigreeSearchRequest = new BrAPIPedigreeSearchRequest(); - pedigreeSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + // TODO: Issue with BrAPI server programDbId filtering, use external refs instead for now + //pedigreeSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + + String extRefId = program.getId().toString(); + String extRefSrc = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS); + pedigreeSearchRequest.addExternalReferenceIdsItem(extRefId); + pedigreeSearchRequest.addExternalReferenceSourcesItem(extRefSrc); + // TODO: add pagination support // .page(page) // .pageSize(pageSize); @@ -63,14 +71,19 @@ public List getPedigree(Program program) throws ApiException PedigreeApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), PedigreeApi.class); - /* - return brAPIDAOUtil.search( + List pedigreeNodes = brAPIDAOUtil.search( api::searchPedigreePost, api::searchPedigreeSearchResultsDbIdGet, pedigreeSearchRequest ); - */ - return new ArrayList<>(); + + // search on external references is id OR source, need to filter for id AND source + pedigreeNodes = pedigreeNodes.stream() + .filter(node -> node.getExternalReferences().stream() + .anyMatch(ref -> ref.getReferenceSource().equals(extRefSrc) && ref.getReferenceId().equals(extRefId))) + .collect(Collectors.toList()); + + return pedigreeNodes; } } \ No newline at end of file From e932d9d94a222d5542514a49a41e6bdb9e7710a9 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 19 Apr 2024 15:56:52 -0400 Subject: [PATCH 11/44] Filtering work --- .../brapi/v2/BrAPIPedigreeController.java | 14 ++++- .../brapi/v2/dao/BrAPIPedigreeDAO.java | 53 ++++++++++++++++++- .../breedinginsight/utilities/Utilities.java | 5 ++ 3 files changed, 68 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java index 1c0b1b5b0..bdc5c1d98 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java @@ -53,7 +53,9 @@ public class BrAPIPedigreeController { private final ProgramService programService; @Inject - public BrAPIPedigreeController(@Property(name = "brapi.server.reference-source") String referenceSource, BrAPIPedigreeDAO pedigreeDAO, ProgramService programService) { + public BrAPIPedigreeController(@Property(name = "brapi.server.reference-source") String referenceSource, + BrAPIPedigreeDAO pedigreeDAO, + ProgramService programService) { this.referenceSource = referenceSource; this.pedigreeDAO = pedigreeDAO; this.programService = programService; @@ -96,7 +98,15 @@ public HttpResponse pedigreeGet(@PathVariable("progra } try { - List pedigree = pedigreeDAO.getPedigree(program.get()); + List pedigree = pedigreeDAO.getPedigree( + program.get(), + Optional.ofNullable(includeParents), + Optional.ofNullable(includeSiblings), + Optional.ofNullable(includeProgeny), + Optional.ofNullable(includeFullTree), + Optional.ofNullable(pedigreeDepth), + Optional.ofNullable(progenyDepth), + Optional.ofNullable(germplasmName)); return HttpResponse.ok( new BrAPIPedigreeListResponse() diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java index 4fa59d805..ae504fbce 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java @@ -52,7 +52,16 @@ public BrAPIPedigreeDAO(ProgramDAO programDAO, BrAPIDAOUtil brAPIDAOUtil, this.referenceSource = referenceSource; } - public List getPedigree(Program program) throws ApiException { + public List getPedigree( + Program program, + Optional includeParents, + Optional includeSiblings, + Optional includeProgeny, + Optional includeFullTree, + Optional pedigreeDepth, + Optional progenyDepth, + Optional germplasmName + ) throws ApiException { // TODO: maybe use get instead of search BrAPIPedigreeSearchRequest pedigreeSearchRequest = new BrAPIPedigreeSearchRequest(); @@ -62,7 +71,15 @@ public List getPedigree(Program program) throws ApiException String extRefId = program.getId().toString(); String extRefSrc = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS); pedigreeSearchRequest.addExternalReferenceIdsItem(extRefId); - pedigreeSearchRequest.addExternalReferenceSourcesItem(extRefSrc); + //pedigreeSearchRequest.addExternalReferenceSourcesItem(extRefSrc); + + includeParents.ifPresent(pedigreeSearchRequest::includeParents); + includeSiblings.ifPresent(pedigreeSearchRequest::includeSiblings); + includeProgeny.ifPresent(pedigreeSearchRequest::includeProgeny); + includeFullTree.ifPresent(pedigreeSearchRequest::includeFullTree); + pedigreeDepth.ifPresent(pedigreeSearchRequest::setPedigreeDepth); + progenyDepth.ifPresent(pedigreeSearchRequest::setProgenyDepth); + germplasmName.ifPresent(pedigreeSearchRequest::addGermplasmNamesItem); // TODO: add pagination support // .page(page) @@ -78,12 +95,44 @@ public List getPedigree(Program program) throws ApiException ); // search on external references is id OR source, need to filter for id AND source + /* pedigreeNodes = pedigreeNodes.stream() .filter(node -> node.getExternalReferences().stream() .anyMatch(ref -> ref.getReferenceSource().equals(extRefSrc) && ref.getReferenceId().equals(extRefId))) .collect(Collectors.toList()); + */ + + //stripProgramKeys(pedigreeNodes, program.getKey()); return pedigreeNodes; } + public List searchPedigree() { + return new ArrayList<>(); + } + + /** + * Removes the program key from the germplasm names in the list of pedigree nodes. + * + * @param pedigreeNodes The list of pedigree nodes. + * @param programKey The program key to be removed. + */ + private void stripProgramKeys(List pedigreeNodes, String programKey) { + pedigreeNodes.forEach(node -> { + node.setGermplasmName(Utilities.removeProgramKeyAnyAccession(node.getGermplasmName(), programKey)); + // TODO: pedigree stripping not right + //node.setPedigreeString(Utilities.removeProgramKeyAnyAccession(node.getPedigreeString(), programKey)); + if (node.getParents() != null) { + node.getParents().forEach(parent -> { + parent.setGermplasmName(Utilities.removeProgramKeyAnyAccession(parent.getGermplasmName(), programKey)); + }); + } + if (node.getProgeny() != null) { + node.getProgeny().forEach(progeny -> { + progeny.setGermplasmName(Utilities.removeProgramKeyAnyAccession(progeny.getGermplasmName(), programKey)); + }); + } + }); + } + } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index c885eaa97..36dacb767 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -87,6 +87,11 @@ public static String removeUnknownProgramKey(String original) { } + public static String removeProgramKeyAnyAccession(String str, String programKey) { + return str.replaceAll("\\[" + programKey + "-.*\\]", "").trim(); + + } + /** * Remove program key from a string. Returns a new value instead of altering original string. * From 1fe76b4f0b0cee485b4a1634a2c86979f86bc9d0 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 19 Apr 2024 16:25:26 -0400 Subject: [PATCH 12/44] Changed to using get call instead of search --- .../brapi/v2/dao/BrAPIPedigreeDAO.java | 113 ++++++++++++++---- 1 file changed, 91 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java index ae504fbce..aa76f5030 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java @@ -18,10 +18,15 @@ package org.breedinginsight.brapi.v2.dao; import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.model.queryParams.germplasm.PedigreeQueryParams; import org.brapi.client.v2.modules.germplasm.PedigreeApi; import org.brapi.v2.model.germ.BrAPIPedigreeNode; import org.brapi.v2.model.germ.request.BrAPIPedigreeSearchRequest; +import org.brapi.v2.model.germ.response.BrAPIPedigreeListResponse; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.Program; @@ -32,9 +37,9 @@ import javax.inject.Inject; import javax.inject.Singleton; import java.util.*; -import java.util.stream.Collectors; @Singleton +@Slf4j public class BrAPIPedigreeDAO { private ProgramDAO programDAO; @@ -52,6 +57,20 @@ public BrAPIPedigreeDAO(ProgramDAO programDAO, BrAPIDAOUtil brAPIDAOUtil, this.referenceSource = referenceSource; } + /** + * Retrieves the pedigree of a given program and optional filters. Used by Helium. TODO: Add rest of parameters + * + * @param program The program for which the pedigree is requested. + * @param includeParents Flag to indicate whether to include parent nodes in the pedigree. (optional) + * @param includeSiblings Flag to indicate whether to include sibling nodes in the pedigree. (optional) + * @param includeProgeny Flag to indicate whether to include progeny nodes in the pedigree. (optional) + * @param includeFullTree Flag to indicate whether to include the full pedigree tree or only immediate ancestors and descendants. (optional) + * @param pedigreeDepth The maximum depth of ancestors and descendants to include in the pedigree. (optional) + * @param progenyDepth The maximum depth of progeny to include in the pedigree. (optional) + * @param germplasmName The name of the germplasm to which the pedigree is limited. (optional) + * @return A list of pedigree nodes representing the pedigree of the program. + * @throws ApiException If an error occurs while making the BrAPI call. + */ public List getPedigree( Program program, Optional includeParents, @@ -62,16 +81,81 @@ public List getPedigree( Optional progenyDepth, Optional germplasmName ) throws ApiException { - // TODO: maybe use get instead of search - BrAPIPedigreeSearchRequest pedigreeSearchRequest = new BrAPIPedigreeSearchRequest(); - // TODO: Issue with BrAPI server programDbId filtering, use external refs instead for now + PedigreeQueryParams pedigreeRequest = new PedigreeQueryParams(); + + // TODO: Issue with BrAPI server programDbId filtering, think germplasm are linked to program through observation + // units and doesn't work if don't have any loaded + // use external refs instead for now //pedigreeSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); String extRefId = program.getId().toString(); String extRefSrc = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS); + pedigreeRequest.externalReferenceId(extRefId); + pedigreeRequest.externalReferenceSource(extRefSrc); + + includeParents.ifPresent(pedigreeRequest::includeParents); + includeSiblings.ifPresent(pedigreeRequest::includeSiblings); + includeProgeny.ifPresent(pedigreeRequest::includeProgeny); + includeFullTree.ifPresent(pedigreeRequest::includeFullTree); + pedigreeDepth.ifPresent(pedigreeRequest::pedigreeDepth); + progenyDepth.ifPresent(pedigreeRequest::progenyDepth); + germplasmName.ifPresent(pedigreeRequest::germplasmName); + // TODO: other parameters + + // TODO: write utility to do paging instead of hardcoding + pedigreeRequest.pageSize(100000); + + ApiResponse brapiPedigree; + try { + brapiPedigree = brAPIEndpointProvider + .get(programDAO.getCoreClient(program.getId()), PedigreeApi.class) + .pedigreeGet(pedigreeRequest); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Error making BrAPI call", e); + } + + List pedigreeNodes = brapiPedigree.getBody().getResult().getData(); + // TODO: once Helium is constructing nodes from DbId we can strip program keys but won't in the mean time + //stripProgramKeys(pedigreeNodes, program.getKey()); + return pedigreeNodes; + } + + /** + * Searches for pedigree nodes based on the given parameters. Not used by Helium TODO: Add rest of parameters + * + * @param program The program to search for pedigree nodes. + * @param includeParents Optional boolean to include parents in the search. + * @param includeSiblings Optional boolean to include siblings in the search. + * @param includeProgeny Optional boolean to include progeny in the search. + * @param includeFullTree Optional boolean to include the full pedigree tree in the search. + * @param pedigreeDepth Optional integer for the maximum depth of the pedigree tree. + * @param progenyDepth Optional integer for the maximum depth of the progeny tree. + * @param germplasmName Optional String to filter the search by germplasm name. + * @return A List of BrAPIPedigreeNode objects that match the search criteria. + * @throws ApiException If an error occurs while searching for pedigree nodes. + */ + public List searchPedigree(Program program, + Optional includeParents, + Optional includeSiblings, + Optional includeProgeny, + Optional includeFullTree, + Optional pedigreeDepth, + Optional progenyDepth, + Optional germplasmName + ) throws ApiException { + + BrAPIPedigreeSearchRequest pedigreeSearchRequest = new BrAPIPedigreeSearchRequest(); + // TODO: Issue with BrAPI server programDbId filtering, think germplasm are linked to program through observation + // units and doesn't work if don't have any loaded + // use external refs instead for now + //pedigreeSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + + // Just use program UUID, shouldn't have any collisions and don't want to get all the germplasm if we were also + // using source because search is OR rather than AND + String extRefId = program.getId().toString(); pedigreeSearchRequest.addExternalReferenceIdsItem(extRefId); - //pedigreeSearchRequest.addExternalReferenceSourcesItem(extRefSrc); includeParents.ifPresent(pedigreeSearchRequest::includeParents); includeSiblings.ifPresent(pedigreeSearchRequest::includeSiblings); @@ -80,10 +164,6 @@ public List getPedigree( pedigreeDepth.ifPresent(pedigreeSearchRequest::setPedigreeDepth); progenyDepth.ifPresent(pedigreeSearchRequest::setProgenyDepth); germplasmName.ifPresent(pedigreeSearchRequest::addGermplasmNamesItem); - - // TODO: add pagination support - // .page(page) - // .pageSize(pageSize); // TODO: other parameters PedigreeApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), PedigreeApi.class); @@ -94,23 +174,12 @@ public List getPedigree( pedigreeSearchRequest ); - // search on external references is id OR source, need to filter for id AND source - /* - pedigreeNodes = pedigreeNodes.stream() - .filter(node -> node.getExternalReferences().stream() - .anyMatch(ref -> ref.getReferenceSource().equals(extRefSrc) && ref.getReferenceId().equals(extRefId))) - .collect(Collectors.toList()); - */ - + // TODO: once Helium is constructing nodes from DbId we can strip program keys but won't in the mean time //stripProgramKeys(pedigreeNodes, program.getKey()); return pedigreeNodes; } - public List searchPedigree() { - return new ArrayList<>(); - } - /** * Removes the program key from the germplasm names in the list of pedigree nodes. * @@ -120,7 +189,7 @@ public List searchPedigree() { private void stripProgramKeys(List pedigreeNodes, String programKey) { pedigreeNodes.forEach(node -> { node.setGermplasmName(Utilities.removeProgramKeyAnyAccession(node.getGermplasmName(), programKey)); - // TODO: pedigree stripping not right + // TODO: pedigree stripping not working right //node.setPedigreeString(Utilities.removeProgramKeyAnyAccession(node.getPedigreeString(), programKey)); if (node.getParents() != null) { node.getParents().forEach(parent -> { From 5a78119bdb6e4828787f4c9039b913d145996f52 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 19 Apr 2024 16:41:30 -0400 Subject: [PATCH 13/44] Added comment --- .../org/breedinginsight/brapi/v2/BrAPIPedigreeController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java index bdc5c1d98..9ebc8d135 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java @@ -137,5 +137,6 @@ public HttpResponse pedigreePut(@PathVariable("programId") UUID programId, @B } // TODO: search and retrieve endpoints + // already have some work done, search call in BrAPIPedigreeDAO } From 4b41a96d96195c0ce136d54525e01b9a2b43e880 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 19 Apr 2024 16:44:28 -0400 Subject: [PATCH 14/44] Commented out unused search call for now, github actions build error --- .../java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java index aa76f5030..3138b6e7a 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java @@ -136,6 +136,7 @@ public List getPedigree( * @return A List of BrAPIPedigreeNode objects that match the search criteria. * @throws ApiException If an error occurs while searching for pedigree nodes. */ + /* public List searchPedigree(Program program, Optional includeParents, Optional includeSiblings, @@ -179,6 +180,7 @@ public List searchPedigree(Program program, return pedigreeNodes; } + */ /** * Removes the program key from the germplasm names in the list of pedigree nodes. From 8d16b80615a4aec7527a442f9b33ba11d05e3260 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 19 Apr 2024 17:05:07 -0400 Subject: [PATCH 15/44] Added comment to utility function --- .../java/org/breedinginsight/utilities/Utilities.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 36dacb767..5cdac442f 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -86,10 +86,15 @@ public static String removeUnknownProgramKey(String original) { return original.replaceAll("\\[.*\\]", "").trim(); } - + /** + * Removes the program key from a string with any accession number. + * + * @param str The string to remove the program key from + * @param programKey The program key to remove + * @return The modified string + */ public static String removeProgramKeyAnyAccession(String str, String programKey) { return str.replaceAll("\\[" + programKey + "-.*\\]", "").trim(); - } /** From b64e7c846bacf33f1f7f469d0793d46a4b1096d7 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 22 Apr 2024 16:44:18 -0400 Subject: [PATCH 16/44] [BI-2101] added transactional email configuration enabled direct SMTP connection to SES provided the correct authentication environment variables are present --- .../breedinginsight/utilities/Utilities.java | 8 +++++++ .../utilities/email/EmailUtil.java | 24 +++++++++++++++---- src/main/resources/application.yml | 2 ++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index c885eaa97..0119009ac 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -252,4 +252,12 @@ private static boolean isSafeChar(char c) { // See https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap03.html#tag_03_282 return Character.isLetterOrDigit(c) || c == '-' || c == '_' || c == '.'; } + + public static boolean isNeitherNullNorEmpty(String s) { + return !isNullOrEmpty(s); + } + + public static boolean isNullOrEmpty(String s) { + return s == null || s.isEmpty(); + } } diff --git a/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java b/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java index f0f6652d9..1db3919a0 100644 --- a/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java +++ b/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java @@ -19,12 +19,10 @@ import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.HttpServerException; +import org.breedinginsight.utilities.Utilities; import javax.inject.Singleton; -import javax.mail.Message; -import javax.mail.MessagingException; -import javax.mail.Session; -import javax.mail.Transport; +import javax.mail.*; import javax.mail.internet.InternetAddress; import javax.mail.internet.MimeMessage; import java.io.UnsupportedEncodingException; @@ -40,13 +38,29 @@ public class EmailUtil { private Integer smtpHostPort; @Property(name = "email.from") private String fromEmail; + @Property(name = "email.relay-server.login") + private String smtpLogin; + @Property(name = "email.relay-server.password") + private String smtpPassword; private Session getSmtpHost() { Properties props = new Properties(); props.put("mail.smtp.host", smtpHostServer); props.put("mail.smtp.port", smtpHostPort); props.put("mail.debug", true); - return Session.getInstance(props, null); + Authenticator auth = null; + if (Utilities.isNeitherNullNorEmpty(smtpLogin) && Utilities.isNeitherNullNorEmpty(smtpPassword)) { + props.put("mail.smtp.auth", true); + props.put("mail.smtp.ssl.trust", smtpHostServer); + props.put("mail.smtp.starttls.enable", true); + auth = new Authenticator() { + @Override + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(smtpLogin, smtpPassword); + } + }; + } + return Session.getInstance(props, auth); } public void sendEmail(String toEmail, String subject, String body){ diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index aec211b82..3613365c3 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -177,6 +177,8 @@ email: relay-server: host: ${EMAIL_RELAY_HOST} port: ${EMAIL_RELAY_PORT} + login: ${EMAIL_RELAY_LOGIN} + password: ${EMAIL_RELAY_PASSWORD} from: ${EMAIL_FROM} redisson: From fe7d0e1cd2a38e7246411a0aa660d4cfbc4b57fb Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 23 Apr 2024 16:45:33 -0400 Subject: [PATCH 17/44] [BI-2101] updated email subject and body --- src/main/java/org/breedinginsight/services/UserService.java | 2 +- src/main/resources/email/newAccountEmail.st | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/services/UserService.java b/src/main/java/org/breedinginsight/services/UserService.java index ebc4e94b4..7502ff8b2 100644 --- a/src/main/java/org/breedinginsight/services/UserService.java +++ b/src/main/java/org/breedinginsight/services/UserService.java @@ -388,7 +388,7 @@ private void sendAccountSignUpEmail(BiUserEntity user, SignedJWT jwtToken) { emailTemplate.add("expiration_time", expirationTime); String filledBody = emailTemplate.render(); - String subject = "New Account Sign Up"; + String subject = "Activate DeltaBreed Account"; // Send email emailUtil.sendEmail(user.getEmail(), subject, filledBody); diff --git a/src/main/resources/email/newAccountEmail.st b/src/main/resources/email/newAccountEmail.st index a5fd73303..6fa9c1154 100644 --- a/src/main/resources/email/newAccountEmail.st +++ b/src/main/resources/email/newAccountEmail.st @@ -2,7 +2,7 @@ Welcome to Breeding Insight! We use a common login system with ORCID to provide authentication and account security. You will need a current ORCID iD and account to log in to Breeding Insight. If you do not already have an ORCID iD, you can create one here: https://orcid.org/register -To activate your Breeding Insight account and connect your ORCID iD to Breeding Insight, use this link: +To activate your DeltaBreed account and connect your ORCID iD to DeltaBreed, use this link: From c5c556e772a0e83fdebf955b8e6f83e8b3a30f57 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 24 Apr 2024 16:05:36 -0400 Subject: [PATCH 18/44] [BI-2101] added default values for email configuration --- src/main/resources/application.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 3613365c3..10acaf1f0 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -177,8 +177,8 @@ email: relay-server: host: ${EMAIL_RELAY_HOST} port: ${EMAIL_RELAY_PORT} - login: ${EMAIL_RELAY_LOGIN} - password: ${EMAIL_RELAY_PASSWORD} + login: ${EMAIL_RELAY_LOGIN:null} + password: ${EMAIL_RELAY_PASSWORD:null} from: ${EMAIL_FROM} redisson: From da45d4be18cd57a980ad649865af920a7ad9a6de Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 30 Apr 2024 11:06:17 -0400 Subject: [PATCH 19/44] [BI-2101] updated .env.template --- .env.template | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.env.template b/.env.template index 0f100a7db..81e26ab06 100644 --- a/.env.template +++ b/.env.template @@ -42,9 +42,11 @@ BRAPI_REFERENCE_SOURCE=breedinginsight.org WEB_BASE_URL=http://localhost:8080 # Email server -EMAIL_RELAY_HOST=mailhog -EMAIL_RELAY_PORT=1025 -EMAIL_FROM=bidevteam@cornell.edu +EMAIL_RELAY_HOST= +EMAIL_RELAY_PORT=<1025 for development, 25 for production> +EMAIL_FROM=noreply@breedinginsight.org +#EMAIL_RELAY_LOGIN= +#EMAIL_RELAY_PASSWORD= GIGWA_HOST= GIGWA_USER= From fa81b8dc379b56044765b1391cf7376fb0483fcc Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 30 Apr 2024 11:06:48 -0400 Subject: [PATCH 20/44] [BI-2101] used existing utility method --- .../java/org/breedinginsight/utilities/Utilities.java | 8 -------- .../org/breedinginsight/utilities/email/EmailUtil.java | 4 ++-- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 0119009ac..c885eaa97 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -252,12 +252,4 @@ private static boolean isSafeChar(char c) { // See https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap03.html#tag_03_282 return Character.isLetterOrDigit(c) || c == '-' || c == '_' || c == '.'; } - - public static boolean isNeitherNullNorEmpty(String s) { - return !isNullOrEmpty(s); - } - - public static boolean isNullOrEmpty(String s) { - return s == null || s.isEmpty(); - } } diff --git a/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java b/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java index 1db3919a0..a839715a1 100644 --- a/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java +++ b/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java @@ -19,7 +19,7 @@ import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.HttpServerException; -import org.breedinginsight.utilities.Utilities; +import org.apache.commons.lang3.StringUtils; import javax.inject.Singleton; import javax.mail.*; @@ -49,7 +49,7 @@ private Session getSmtpHost() { props.put("mail.smtp.port", smtpHostPort); props.put("mail.debug", true); Authenticator auth = null; - if (Utilities.isNeitherNullNorEmpty(smtpLogin) && Utilities.isNeitherNullNorEmpty(smtpPassword)) { + if (StringUtils.isNotBlank(smtpLogin) && StringUtils.isNotBlank(smtpPassword)) { props.put("mail.smtp.auth", true); props.put("mail.smtp.ssl.trust", smtpHostServer); props.put("mail.smtp.starttls.enable", true); From 20eda41f88a21745a1fc7865645443b0f67bf1d0 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 1 May 2024 09:19:51 -0400 Subject: [PATCH 21/44] [BI-2104] protect against null pointer exception --- .../importer/services/processors/ExperimentProcessor.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 45d26f7cb..062857af7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1175,7 +1175,10 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra } boolean isTimestampMatched(String observationHash, String timeStamp) { - OffsetDateTime priorStamp = existingObsByObsHash.get(observationHash).getObservationTimeStamp(); + OffsetDateTime priorStamp = null; + if(existingObsByObsHash.get(observationHash)!=null){ + priorStamp = existingObsByObsHash.get(observationHash).getObservationTimeStamp(); + } if (priorStamp == null) { return timeStamp == null; } From 5b26088890d29b447bd84563933f332c300ba427 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Thu, 2 May 2024 09:45:39 -0400 Subject: [PATCH 22/44] [BI-2065] - Code review timestamp fix Handled case where present observation but blank associated timestamp led to an unhandled attempt to parse said blank timestamp --- .../importer/services/processors/ExperimentProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index ec7588fa4..f01578e21 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1111,7 +1111,7 @@ private void fetchOrCreateObservationPIO(Program program, newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); if (!isValueMatched(key, value)){ newObservation.setValue(value); - } else if (!isTimestampMatched(key, timeStampValue)) { + } else if (!StringUtils.isBlank(timeStampValue) && !isTimestampMatched(key, timeStampValue)) { DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timeStampValue)); newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); From fa114becf9b810faa7f0daced2e6a8c297555885 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 2 May 2024 14:20:16 +0000 Subject: [PATCH 23/44] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index e25a7e336..8733b71ff 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0+723 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c936dc56136e0006616bb5d713434110100a4cb4 +version=v0.10.0+725 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/9375820014de664ae9ada681ab4ab075e2b82ebc From d283ded97c4ac665ea9ff19f00c619b82758f405 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 2 May 2024 14:09:09 -0400 Subject: [PATCH 24/44] Removed unused reference source --- .../breedinginsight/brapi/v2/BrAPIPedigreeController.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java index 9ebc8d135..51799b346 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java @@ -17,7 +17,6 @@ package org.breedinginsight.brapi.v2; -import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; import io.micronaut.http.annotation.*; @@ -46,17 +45,13 @@ @Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) @Secured(SecurityRule.IS_AUTHENTICATED) public class BrAPIPedigreeController { - private final String referenceSource; - private final BrAPIPedigreeDAO pedigreeDAO; private final ProgramService programService; @Inject - public BrAPIPedigreeController(@Property(name = "brapi.server.reference-source") String referenceSource, - BrAPIPedigreeDAO pedigreeDAO, + public BrAPIPedigreeController(BrAPIPedigreeDAO pedigreeDAO, ProgramService programService) { - this.referenceSource = referenceSource; this.pedigreeDAO = pedigreeDAO; this.programService = programService; } From 5b43b9fceb5a41e2651d616cf59ff5bd707d1d26 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 2 May 2024 14:21:38 -0400 Subject: [PATCH 25/44] Added details to method documentation justifying commented code --- .../org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java index 3138b6e7a..259e9cf81 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java @@ -123,7 +123,11 @@ public List getPedigree( } /** - * Searches for pedigree nodes based on the given parameters. Not used by Helium TODO: Add rest of parameters + * Searches for pedigree nodes based on the given parameters. Not used by Helium but keeping commented out for + * now in case we want to implement the search endpoints in the future, work here has already been started to + * support that. + * + * TODO: Add rest of parameters * * @param program The program to search for pedigree nodes. * @param includeParents Optional boolean to include parents in the search. From a113a49e39c717d53eb9999711d9a0c3bbd89ef0 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 2 May 2024 14:24:55 -0400 Subject: [PATCH 26/44] Added documentation for stripProgramKeys --- .../org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java index 259e9cf81..a00957f2c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java @@ -187,7 +187,9 @@ public List searchPedigree(Program program, */ /** - * Removes the program key from the germplasm names in the list of pedigree nodes. + * Removes the program key from the germplasm names in the list of pedigree nodes. Not used currently but will be in + * future if we decide to strip the program keys when Helium has been updated to use the germplasmDbId for uniqueness + * rather than germplasmName. * * @param pedigreeNodes The list of pedigree nodes. * @param programKey The program key to be removed. From 22abf884314d0be719096e589b863e3a27e664f8 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 3 May 2024 16:55:08 +0000 Subject: [PATCH 27/44] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 8733b71ff..2d7c7da2b 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0+725 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/9375820014de664ae9ada681ab4ab075e2b82ebc +version=v0.10.0+727 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/d36bcce835cad2539501e2395c39dab84bf12a6c From 480c3b7791584f425970fb7e00950c11847ae0eb Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 3 May 2024 18:10:07 +0000 Subject: [PATCH 28/44] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 2d7c7da2b..4b9b4d2ac 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0+727 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/d36bcce835cad2539501e2395c39dab84bf12a6c +version=v0.10.0+729 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/2be3bf139db0bc693f8d51d541383cfa8e2b506f From c18b75ad2ec60fef9ac7510de0059885cb818702 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 6 May 2024 15:33:00 +0000 Subject: [PATCH 29/44] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 4b9b4d2ac..c580c09a5 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0+729 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/2be3bf139db0bc693f8d51d541383cfa8e2b506f +version=v0.10.0+731 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/480c3b7791584f425970fb7e00950c11847ae0eb From e26da32b3636a13d98fec750c041d517dbce6b88 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 6 May 2024 11:48:13 -0400 Subject: [PATCH 30/44] Update version.properties --- src/main/resources/version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index c580c09a5..02f8f5ace 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0+731 +version=v0.9.1+731 versionInfo=https://github.com/Breeding-Insight/bi-api/commit/480c3b7791584f425970fb7e00950c11847ae0eb From 8a95cda598da5c2b179075a175edb62db4978a9f Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 6 May 2024 15:48:26 +0000 Subject: [PATCH 31/44] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 02f8f5ace..694f986e0 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.1+731 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/480c3b7791584f425970fb7e00950c11847ae0eb +version=v0.9.1+733 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/e26da32b3636a13d98fec750c041d517dbce6b88 From 6a0278cbf85dae2d57ff5b286000dec320b7de78 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 6 May 2024 20:22:04 +0000 Subject: [PATCH 32/44] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 4b9b4d2ac..04625f30d 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0+729 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/2be3bf139db0bc693f8d51d541383cfa8e2b506f +version=v0.10.0+735 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/38f35d01af86d4af581d3ffaa54d65d5521ee5fe From 72bf86b660d7fcb7b0b0d3479a8fdc72f64e1a5f Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 7 May 2024 09:53:41 -0400 Subject: [PATCH 33/44] [BI-2101-fix] updated SMTP port in template --- .env.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.template b/.env.template index 81e26ab06..407b99d50 100644 --- a/.env.template +++ b/.env.template @@ -43,7 +43,7 @@ WEB_BASE_URL=http://localhost:8080 # Email server EMAIL_RELAY_HOST= -EMAIL_RELAY_PORT=<1025 for development, 25 for production> +EMAIL_RELAY_PORT=<1025 for development, 2587 for production> EMAIL_FROM=noreply@breedinginsight.org #EMAIL_RELAY_LOGIN= #EMAIL_RELAY_PASSWORD= From 2225009e732f63be4b82ca157e9cda7708fe2a1c Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 7 May 2024 13:54:15 +0000 Subject: [PATCH 34/44] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 04625f30d..50c54602e 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0+735 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/38f35d01af86d4af581d3ffaa54d65d5521ee5fe +version=v0.10.0+737 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/72bf86b660d7fcb7b0b0d3479a8fdc72f64e1a5f From 86922f289bb6f8ef74768a1d2607259e0b0e351e Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 8 May 2024 18:36:09 +0000 Subject: [PATCH 35/44] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 694f986e0..93a4ec11d 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.1+733 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/e26da32b3636a13d98fec750c041d517dbce6b88 +version=v0.9.1+739 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/ed1fd023abd6980fb4af05e801cfafb11a2b381e From e7e46b70afc2f6a6b0acdee36ad8e0bc4e01299f Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 13 May 2024 15:12:37 -0400 Subject: [PATCH 36/44] [BI-2127] return Lat, Long, Elevation, and RTK --- .../brapi/v2/services/BrAPITrialService.java | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index caab88879..e669e46df 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,5 +1,9 @@ package org.breedinginsight.brapi.v2.services; +import com.github.filosganga.geogson.model.Coordinates; +import com.github.filosganga.geogson.model.Point; +import com.github.filosganga.geogson.model.positions.SinglePosition; +import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; import io.micronaut.http.MediaType; import io.micronaut.http.server.exceptions.InternalServerException; @@ -465,10 +469,19 @@ private Map createExportRow( row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); row.put(ExperimentObservation.Columns.ENV, Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); + + Coordinates coordinates = extractCoordinates(ou); + row.put( ExperimentObservation.Columns.LAT, coordinates==null? null : String.valueOf(coordinates.getLat()) ); + row.put( ExperimentObservation.Columns.LONG, coordinates==null? null : String.valueOf(coordinates.getLon()) ); + row.put( ExperimentObservation.Columns.ELEVATION, coordinates==null? null : String.valueOf(coordinates.getAlt()) ); + BrAPISeason season = seasonDAO.getSeasonById(study.getSeasons().get(0), program.getId()); row.put(ExperimentObservation.Columns.ENV_YEAR, season.getYear()); row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + JsonObject additionalInfo = ou.getAdditionalInfo(); + row.put(ExperimentObservation.Columns.RTK, additionalInfo.get(BrAPIAdditionalInfoFields.RTK).getAsString()); + // get replicate number Optional repLevel = ou.getObservationUnitPosition() .getObservationLevelRelationships().stream() @@ -484,9 +497,12 @@ private Map createExportRow( .findFirst(); blockLevel.ifPresent(brAPIObservationUnitLevelRelationship -> row.put(ExperimentObservation.Columns.BLOCK_NUM, Integer.parseInt(brAPIObservationUnitLevelRelationship.getLevelCode()))); - if (ou.getObservationUnitPosition() != null && ou.getObservationUnitPosition().getPositionCoordinateX() != null && - ou.getObservationUnitPosition().getPositionCoordinateY() != null) { + if (ou.getObservationUnitPosition() != null && ou.getObservationUnitPosition().getPositionCoordinateX() != null + ) { row.put(ExperimentObservation.Columns.ROW, ou.getObservationUnitPosition().getPositionCoordinateX()); + } + if (ou.getObservationUnitPosition() != null && + ou.getObservationUnitPosition().getPositionCoordinateY() != null) { row.put(ExperimentObservation.Columns.COLUMN, ou.getObservationUnitPosition().getPositionCoordinateY()); } if (ou.getTreatments() != null && !ou.getTreatments().isEmpty()) { @@ -499,7 +515,22 @@ private Map createExportRow( return row; } - + private Coordinates extractCoordinates(BrAPIObservationUnit ou){ + Coordinates coordinates = null; + if ( ou.getObservationUnitPosition()!=null + && ou.getObservationUnitPosition().getGeoCoordinates()!=null + && ou.getObservationUnitPosition().getGeoCoordinates().getGeometry()!=null + && ou.getObservationUnitPosition().getGeoCoordinates().getGeometry().positions()!=null + ) + { + Object o = ou.getObservationUnitPosition().getGeoCoordinates().getGeometry().positions(); + if (o instanceof SinglePosition){ + SinglePosition sp = (SinglePosition)o; + coordinates= sp.coordinates(); + } + } + return coordinates; + } private void addObsVarColumns( List columns, From 1a909b6131f6de47b60e177da515e4336d33b904 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 13 May 2024 16:03:10 -0400 Subject: [PATCH 37/44] [BI-2127] protect against nulls --- .../brapi/v2/services/BrAPITrialService.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index e669e46df..381514182 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,7 +1,6 @@ package org.breedinginsight.brapi.v2.services; import com.github.filosganga.geogson.model.Coordinates; -import com.github.filosganga.geogson.model.Point; import com.github.filosganga.geogson.model.positions.SinglePosition; import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; @@ -470,18 +469,23 @@ private Map createExportRow( row.put(ExperimentObservation.Columns.ENV, Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); + // Lat, Long, Elevation Coordinates coordinates = extractCoordinates(ou); row.put( ExperimentObservation.Columns.LAT, coordinates==null? null : String.valueOf(coordinates.getLat()) ); row.put( ExperimentObservation.Columns.LONG, coordinates==null? null : String.valueOf(coordinates.getLon()) ); row.put( ExperimentObservation.Columns.ELEVATION, coordinates==null? null : String.valueOf(coordinates.getAlt()) ); + // RTK + JsonObject additionalInfo = ou.getAdditionalInfo(); + String rtk = ( additionalInfo==null || additionalInfo.get(BrAPIAdditionalInfoFields.RTK) ==null ) + ? null + : additionalInfo.get(BrAPIAdditionalInfoFields.RTK).getAsString(); + row.put(ExperimentObservation.Columns.RTK, rtk); + BrAPISeason season = seasonDAO.getSeasonById(study.getSeasons().get(0), program.getId()); row.put(ExperimentObservation.Columns.ENV_YEAR, season.getYear()); row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); - JsonObject additionalInfo = ou.getAdditionalInfo(); - row.put(ExperimentObservation.Columns.RTK, additionalInfo.get(BrAPIAdditionalInfoFields.RTK).getAsString()); - // get replicate number Optional repLevel = ou.getObservationUnitPosition() .getObservationLevelRelationships().stream() @@ -497,14 +501,13 @@ private Map createExportRow( .findFirst(); blockLevel.ifPresent(brAPIObservationUnitLevelRelationship -> row.put(ExperimentObservation.Columns.BLOCK_NUM, Integer.parseInt(brAPIObservationUnitLevelRelationship.getLevelCode()))); - if (ou.getObservationUnitPosition() != null && ou.getObservationUnitPosition().getPositionCoordinateX() != null - ) { + + //Row and Column + if ( ou.getObservationUnitPosition() != null ) { row.put(ExperimentObservation.Columns.ROW, ou.getObservationUnitPosition().getPositionCoordinateX()); - } - if (ou.getObservationUnitPosition() != null && - ou.getObservationUnitPosition().getPositionCoordinateY() != null) { row.put(ExperimentObservation.Columns.COLUMN, ou.getObservationUnitPosition().getPositionCoordinateY()); } + if (ou.getTreatments() != null && !ou.getTreatments().isEmpty()) { row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, ou.getTreatments().get(0).getFactor()); } else { From 5a2100cbe6272636597fe67e7e175b8bf342113d Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 14 May 2024 10:24:28 -0400 Subject: [PATCH 38/44] [BI-2127] protect against a double beign NaN (not yet set) before converting to a string --- .../brapi/v2/services/BrAPITrialService.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 381514182..7c18824e7 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -471,9 +471,9 @@ private Map createExportRow( // Lat, Long, Elevation Coordinates coordinates = extractCoordinates(ou); - row.put( ExperimentObservation.Columns.LAT, coordinates==null? null : String.valueOf(coordinates.getLat()) ); - row.put( ExperimentObservation.Columns.LONG, coordinates==null? null : String.valueOf(coordinates.getLon()) ); - row.put( ExperimentObservation.Columns.ELEVATION, coordinates==null? null : String.valueOf(coordinates.getAlt()) ); + row.put( ExperimentObservation.Columns.LAT, coordinates==null? null : doubleToString(coordinates.getLat()) ); + row.put( ExperimentObservation.Columns.LONG, coordinates==null? null : doubleToString(coordinates.getLon()) ); + row.put( ExperimentObservation.Columns.ELEVATION, coordinates==null? null : doubleToString(coordinates.getAlt()) ); // RTK JsonObject additionalInfo = ou.getAdditionalInfo(); @@ -518,6 +518,9 @@ private Map createExportRow( return row; } + private String doubleToString(double val){ + return Double.isNaN(val) ? null : String.valueOf( val ); + } private Coordinates extractCoordinates(BrAPIObservationUnit ou){ Coordinates coordinates = null; if ( ou.getObservationUnitPosition()!=null From a52eebd51c245227883849524d9809a2be0b96ad Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 15 May 2024 12:04:45 -0400 Subject: [PATCH 39/44] [BI-2128] implemented fix and integration tests --- .../processors/ExperimentProcessor.java | 3 +- .../importer/ExperimentFileImportTest.java | 96 +++++++++++++++++-- 2 files changed, 89 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index f27d89338..95e196c5b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1244,7 +1244,8 @@ private void fetchOrCreateObservationPIO(Program program, } if (existingObsByObsHash.containsKey(key)) { - if (!isObservationMatched(key, value, column, rowNum)){ + // Update observation value only if it is changed and new value is not blank. + if (!isObservationMatched(key, value, column, rowNum) && StringUtils.isNotBlank(value)){ // prior observation with updated value newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 6bc203d64..09953b2f7 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -85,7 +85,6 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class ExperimentFileImportTest extends BrAPITest { - private static final String OVERWRITE = "overwrite"; private FannyPack securityFp; private String mappingId; @@ -822,6 +821,75 @@ public void importNewObservationDataByObsUnitId(boolean commit) { } } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void verifyBlankObsInOverwriteIsNoOp(boolean commit) { + List traits = importTestUtils.createTraits(1); + Program program = createProgram("Overwrite Attempt With Blank Obs"+(commit ? "C" : "P"), "NOOP"+(commit ? "C" : "P"), "NOOP"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), "1"); // Valid observation value. + + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + + // Fetch the ObsUnitId to use in the overwrite upload. + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); + Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); + assertTrue(trialIdXref.isPresent()); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); + Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); + assertTrue(ouIdXref.isPresent()); + + assertRowSaved(newExp, program, traits); + + Map newObsVar = new HashMap<>(); + newObsVar.put(Columns.GERMPLASM_GID, "1"); + newObsVar.put(Columns.TEST_CHECK, "T"); + newObsVar.put(Columns.EXP_TITLE, "Test Exp"); + newObsVar.put(Columns.EXP_UNIT, "Plot"); + newObsVar.put(Columns.EXP_TYPE, "Phenotyping"); + newObsVar.put(Columns.ENV, "New Env"); + newObsVar.put(Columns.ENV_LOCATION, "Location A"); + newObsVar.put(Columns.ENV_YEAR, "2023"); + newObsVar.put(Columns.EXP_UNIT_ID, "a-1"); + newObsVar.put(Columns.REP_NUM, "1"); + newObsVar.put(Columns.BLOCK_NUM, "1"); + newObsVar.put(Columns.ROW, "1"); + newObsVar.put(Columns.COLUMN, "1"); + newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); // Indicates this is an overwrite. + newObsVar.put(traits.get(0).getObservationVariableName(), ""); // Empty string should be no op. + + Map requestBody = new HashMap<>(); + requestBody.put("overwrite", "true"); + requestBody.put("overwriteReason", "testing"); + + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), requestBody, commit, client, program, mappingId); + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + // Verify that the overwrite attempt with blank observation value did not overwrite the original value. + assertRowSaved(newExp, program, traits); + if(commit) { + assertRowSaved(newExp, program, traits); + } else { + assertValidPreviewRow(newExp, row, program, traits); + } + } @ParameterizedTest @ValueSource(booleans = {true, false}) @@ -1086,15 +1154,15 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { /* Scenario: - - an experiment was created with observations - - do a second upload with additional observations for the experiment. Make sure the original observations are blank values - - verify the second set of observations get uploaded successfully + - Create an experiment with valid observations. + - Upload a second file with (1) a blank observation, (2) a changed valid observation, and (3) a new observation for the experiment. + - Verify that (1) the blank observation makes no change, (2) the changed observation is overwritten, and (3) new observations are appended to the experiment. */ @ParameterizedTest @ValueSource(booleans = {true, false}) @SneakyThrows public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { - List traits = importTestUtils.createTraits(2); + List traits = importTestUtils.createTraits(3); Program program = createProgram("Exp with additional Uploads (blank) "+(commit ? "C" : "P"), "EXAUB"+(commit ? "C" : "P"), "EXAUB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); @@ -1110,7 +1178,9 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newExp.put(Columns.BLOCK_NUM, "1"); newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - newExp.put(traits.get(0).getObservationVariableName(), "1"); + String originalValue = "1"; // Convenience variable, this value is reused. + newExp.put(traits.get(0).getObservationVariableName(), originalValue); + newExp.put(traits.get(1).getObservationVariableName(), "2"); importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); @@ -1140,10 +1210,15 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); - newObservation.put(traits.get(0).getObservationVariableName(), ""); - newObservation.put(traits.get(1).getObservationVariableName(), "2"); + newObservation.put(traits.get(0).getObservationVariableName(), ""); // This blank value should not overwrite. + newObservation.put(traits.get(1).getObservationVariableName(), "3"); // This valid value should overwrite. + newObservation.put(traits.get(2).getObservationVariableName(), "4"); // This valid new observation should be appended. - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + Map requestBody = new HashMap<>(); + requestBody.put("overwrite", "true"); + requestBody.put("overwriteReason", "testing"); + + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), requestBody, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -1154,6 +1229,9 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); + // The blank value should not have produced an overwrite. + // This change is to make the "expected" value correct. + newObservation.put(traits.get(0).getObservationVariableName(), originalValue); if(commit) { assertRowSaved(newObservation, program, traits); } else { From c3af14fe0e56d8f42d86667ec06708f254099c60 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 15 May 2024 12:14:10 -0400 Subject: [PATCH 40/44] [BI-2128] added test description --- .../brapps/importer/ExperimentFileImportTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 09953b2f7..2d884bac8 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -821,6 +821,12 @@ public void importNewObservationDataByObsUnitId(boolean commit) { } } + /* + Scenario: + - an experiment was created with observations + - an overwrite operation is attempted with blank observation values + - verify blank obervation values do not overwrite original values + */ @ParameterizedTest @ValueSource(booleans = {true, false}) @SneakyThrows From fa8c7cb586f4bf4110a505e10e090fecee0cd714 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 15 May 2024 13:11:44 -0400 Subject: [PATCH 41/44] [BI-2128] fixed typo --- .../brapps/importer/ExperimentFileImportTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 2d884bac8..48bffd4c2 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -825,7 +825,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { Scenario: - an experiment was created with observations - an overwrite operation is attempted with blank observation values - - verify blank obervation values do not overwrite original values + - verify blank observation values do not overwrite original values */ @ParameterizedTest @ValueSource(booleans = {true, false}) From c67e67aac2b91c6ff95748b930c0979faf468c1e Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 16 May 2024 13:47:38 +0000 Subject: [PATCH 42/44] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 50c54602e..f1fe6063d 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0+737 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/72bf86b660d7fcb7b0b0d3479a8fdc72f64e1a5f +version=v0.10.0+743 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/626d73785f28db9b297e0e260a635e76786a7868 From 2e41d4c75b49e7988a660d51495e21a02cd5fd81 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 17 May 2024 16:26:40 -0400 Subject: [PATCH 43/44] [BI-2127] made code more readable --- .../brapi/v2/services/BrAPITrialService.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 7c18824e7..dd9bde018 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -471,9 +471,15 @@ private Map createExportRow( // Lat, Long, Elevation Coordinates coordinates = extractCoordinates(ou); - row.put( ExperimentObservation.Columns.LAT, coordinates==null? null : doubleToString(coordinates.getLat()) ); - row.put( ExperimentObservation.Columns.LONG, coordinates==null? null : doubleToString(coordinates.getLon()) ); - row.put( ExperimentObservation.Columns.ELEVATION, coordinates==null? null : doubleToString(coordinates.getAlt()) ); + Optional.ofNullable(coordinates) + .map(c -> doubleToString(c.getLat())) + .ifPresent(lat -> row.put(ExperimentObservation.Columns.LAT, lat)); + Optional.ofNullable(coordinates) + .map(c -> doubleToString(c.getLon())) + .ifPresent(lon -> row.put(ExperimentObservation.Columns.LONG, lon)); + Optional.ofNullable(coordinates) + .map(c -> doubleToString(c.getAlt())) + .ifPresent(elevation -> row.put(ExperimentObservation.Columns.ELEVATION, elevation)); // RTK JsonObject additionalInfo = ou.getAdditionalInfo(); From 453cf79701e6a6094a4931091470426f5fa81bff Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 22 May 2024 16:03:38 +0000 Subject: [PATCH 44/44] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index f1fe6063d..b8a9bf8f8 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0+743 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/626d73785f28db9b297e0e260a635e76786a7868 +version=v0.10.0+745 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6662651d581bb80396250f3ff38bc9443c4a4744