diff --git a/src/main/java/org/breedinginsight/api/v1/controller/ProgramController.java b/src/main/java/org/breedinginsight/api/v1/controller/ProgramController.java index 43c24aa52..39b046cd5 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/ProgramController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ProgramController.java @@ -24,6 +24,7 @@ import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.api.auth.*; import org.breedinginsight.api.model.v1.request.*; import org.breedinginsight.api.model.v1.request.query.QueryParams; @@ -37,6 +38,7 @@ import org.breedinginsight.model.*; import org.breedinginsight.services.ProgramObservationLevelService; import org.breedinginsight.services.exceptions.*; +import org.breedinginsight.utilities.Utilities; import org.breedinginsight.utilities.response.mappers.ProgramLocationQueryMapper; import org.breedinginsight.utilities.response.mappers.ProgramQueryMapper; import org.breedinginsight.api.model.v1.response.DataResponse; @@ -302,6 +304,9 @@ public HttpResponse>> getProgramLocations } catch (DoesNotExistException e){ log.info(e.getMessage()); return HttpResponse.notFound(); + } catch (ApiException e) { + log.error("Error fetching program locations: " + Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); } } @@ -318,8 +323,11 @@ public HttpResponse>> postProgramLocation return ResponseUtils.getQueryResponse(programLocations, programLocationQueryMapper, searchRequest, queryParams); } catch (DoesNotExistException e){ - log.info(e.getMessage()); + log.error(e.getMessage(), e); return HttpResponse.notFound(); + } catch (ApiException e) { + log.error("Error fetching program locations: " + Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); } } @@ -332,7 +340,13 @@ public HttpResponse>> postProgramLocation public HttpResponse> getProgramLocations(@PathVariable UUID programId, @PathVariable UUID locationId) { - Optional programLocation = programLocationService.getById(programId, locationId); + Optional programLocation = null; + try { + programLocation = programLocationService.getById(programId, locationId); + } catch (ApiException e) { + log.error("Error fetching program location: " + Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); + } if(programLocation.isPresent()) { Response response = new Response(programLocation.get()); diff --git a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java b/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java index d185ee3f3..60ee776d0 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java @@ -30,6 +30,7 @@ import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapi.v2.model.request.query.GermplasmQuery; +import org.breedinginsight.utilities.Utilities; import org.breedinginsight.utilities.response.mappers.GermplasmQueryMapper; import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; import org.breedinginsight.brapps.importer.model.exports.FileType; @@ -44,10 +45,7 @@ import javax.inject.Inject; import javax.validation.Valid; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.UUID; +import java.util.*; @Slf4j @Controller("/${micronaut.bi.api.version}") @@ -205,7 +203,9 @@ public HttpResponse getGermplasmPedigreeInfo( metadata.setPagination(pagination); response = new BrAPIGermplasmPedigreeResponse(); } else { - BrAPIGermplasm germplasm = germplasmService.getGermplasmByDBID(programId, germplasmId); + BrAPIGermplasm germplasm = germplasmService.getGermplasmByDBID(programId, germplasmId) + .orElseThrow(() -> new DoesNotExistException("DBID for this germplasm does not exist")); + //Forward the pedigree call to the backing BrAPI system of the program passing the germplasmDbId that came in the request GermplasmApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), GermplasmApi.class); ApiResponse pedigreeResponse = api.germplasmGermplasmDbIdPedigreeGet(germplasmId, notation, includeSiblings); @@ -238,12 +238,15 @@ public HttpResponse getGermplasmPedigreeInfo( response.setResult(returnNode); response.setMetadata(metadata); return HttpResponse.ok(response); - } catch (InternalServerException | ApiException e) { + } catch (InternalServerException e) { log.error(e.getMessage(), e); return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving pedigree node"); } catch (DoesNotExistException e) { log.error(e.getMessage(), e); return HttpResponse.status(HttpStatus.NOT_FOUND, "Pedigree node not found"); + } catch (ApiException e) { + log.info(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving pedigree node"); } } @@ -302,8 +305,11 @@ public HttpResponse getGermplasmProgenyInfo( } return HttpResponse.ok(progenyResponse.getBody()); } - } catch (InternalServerException | ApiException e) { - log.info(e.getMessage(), e); + } catch (InternalServerException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving pedigree node"); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving pedigree node"); } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index 280e31bf3..1f5c83e18 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -273,15 +273,22 @@ public BrAPIGermplasm getGermplasmByUUID(String germplasmId, UUID programId) thr return germplasm; } - public BrAPIGermplasm getGermplasmByDBID(String germplasmDbId, UUID programId) throws ApiException, DoesNotExistException { + public Optional getGermplasmByDBID(String germplasmDbId, UUID programId) throws ApiException { Map cache = programGermplasmCache.get(programId); //key is UUID, want to filter by DBID BrAPIGermplasm germplasm = null; if (cache != null) { germplasm = cache.values().stream().filter(x -> x.getGermplasmDbId().equals(germplasmDbId)).collect(Collectors.toList()).get(0); } - if (germplasm == null) { - throw new DoesNotExistException("DBID for this germplasm does not exist"); + return Optional.ofNullable(germplasm); + } + + public List getGermplasmsByDBID(Collection germplasmDbIds, UUID programId) throws ApiException { + Map cache = programGermplasmCache.get(programId); + //key is UUID, want to filter by DBID + List germplasm = new ArrayList<>(); + if (cache != null) { + germplasm = cache.values().stream().filter(x -> germplasmDbIds.contains(x.getGermplasmDbId())).collect(Collectors.toList()); } return germplasm; } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java index 03451b43b..d09848c91 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java @@ -73,12 +73,8 @@ public BrAPIGermplasm getGermplasmByUUID(UUID programId, String germplasmId) thr } } - public BrAPIGermplasm getGermplasmByDBID(UUID programId, String germplasmId) throws DoesNotExistException { - try { - return germplasmDAO.getGermplasmByDBID(germplasmId, programId); - } catch (ApiException e) { - throw new InternalServerException(e.getMessage(), e); - } + public Optional getGermplasmByDBID(UUID programId, String germplasmId) throws ApiException { + return germplasmDAO.getGermplasmByDBID(germplasmId, programId); } public List getGermplasmListsByProgramId(UUID programId, HttpRequest request) throws DoesNotExistException, ApiException { diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java index d04b05b43..4a7eb90e5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java @@ -21,7 +21,9 @@ import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; +import javax.validation.constraints.NotNull; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.UUID; @@ -42,6 +44,10 @@ public BrAPIListDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brA } public List getListByName(List listNames, UUID programId) throws ApiException { + if(listNames.isEmpty()) { + return Collections.emptyList(); + } + BrAPIListSearchRequest listSearch = new BrAPIListSearchRequest(); listSearch.listNames(listNames); ListsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ListsApi.class); @@ -58,7 +64,7 @@ public BrAPIListsSingleResponse getListById(String listId, UUID programId) throw return response.getBody(); } - public List getListByTypeAndExternalRef(BrAPIListTypes listType, UUID programId, String externalReferenceSource, UUID externalReferenceId) throws ApiException { + public List getListByTypeAndExternalRef(@NotNull BrAPIListTypes listType, UUID programId, String externalReferenceSource, UUID externalReferenceId) throws ApiException { BrAPIListSearchRequest searchRequest = new BrAPIListSearchRequest() .externalReferenceIDs(List.of(externalReferenceId.toString())) .externalReferenceSources(List.of(externalReferenceSource)) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java index 65ddeda54..2fe7b8b01 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java @@ -28,9 +28,7 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; +import java.util.*; @Singleton public class BrAPILocationDAO { @@ -49,6 +47,9 @@ public BrAPILocationDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil } public List getLocationsByName(List locationNames, UUID programId) throws ApiException { + if(locationNames.isEmpty()) { + return Collections.emptyList(); + } BrAPILocationSearchRequest locationSearchRequest = new BrAPILocationSearchRequest(); locationSearchRequest.setLocationNames(new ArrayList<>(locationNames)); @@ -61,9 +62,24 @@ public List getLocationsByName(List locationNames, UUID p ); } - public List createBrAPILocation(List brAPILocationList, UUID programId, ImportUpload upload) throws ApiException { + public List createBrAPILocations(List brAPILocationList, UUID programId, ImportUpload upload) throws ApiException { LocationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), LocationsApi.class); return brAPIDAOUtil.post(brAPILocationList, upload, api::locationsPost, importDAO::update); } + public List getLocationsByDbId(Collection locationDbIds, UUID programId) throws ApiException { + if(locationDbIds.isEmpty()) { + return Collections.emptyList(); + } + + BrAPILocationSearchRequest locationSearchRequest = new BrAPILocationSearchRequest(); + locationSearchRequest.setLocationDbIds(new ArrayList<>(locationDbIds)); + //TODO: Locations don't connect to programs. How to get locations for the program? + LocationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), LocationsApi.class); + return brAPIDAOUtil.search( + api::searchLocationsPost, + api::searchLocationsSearchResultsDbIdGet, + locationSearchRequest + ); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java index 1115b625c..ecaa4bff0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java @@ -25,7 +25,6 @@ import org.brapi.v2.model.pheno.request.BrAPIObservationSearchRequest; import org.brapi.v2.model.pheno.response.BrAPIObservationListResponse; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.daos.ObservationDAO; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.Program; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; @@ -34,10 +33,7 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.UUID; +import java.util.*; import static org.brapi.v2.model.BrAPIWSMIMEDataTypes.APPLICATION_JSON; @@ -58,6 +54,9 @@ public BrAPIObservationDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOU } public List getObservationsByStudyName(List studyNames, Program program) throws ApiException { + if(studyNames.isEmpty()) { + return Collections.emptyList(); + } BrAPIObservationSearchRequest observationSearchRequest = new BrAPIObservationSearchRequest(); observationSearchRequest.setProgramDbIds(List.of(program.getBrapiProgram().getProgramDbId())); @@ -70,14 +69,30 @@ public List getObservationsByStudyName(List studyNames ); } + public List getObservationsByObservationUnitsAndVariables(Collection ouDbIds, Collection variableDbIds, Program program) throws ApiException { + if(ouDbIds.isEmpty() || variableDbIds.isEmpty()) { + return Collections.emptyList(); + } + + BrAPIObservationSearchRequest observationSearchRequest = new BrAPIObservationSearchRequest(); + observationSearchRequest.setProgramDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + observationSearchRequest.setObservationUnitDbIds(new ArrayList<>(ouDbIds)); + observationSearchRequest.setObservationVariableDbIds(new ArrayList<>(variableDbIds)); + ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationsApi.class); + return brAPIDAOUtil.search( + api::searchObservationsPost, + (brAPIWSMIMEDataTypes, searchResultsDbId, page, pageSize) -> searchObservationsSearchResultsDbIdGet(program.getId(), searchResultsDbId, page, pageSize), + observationSearchRequest + ); + } + @NotNull - private ApiResponse, Optional>> - searchObservationsSearchResultsDbIdGet(UUID programId, String searchResultsDbId, Integer page, Integer pageSize) throws ApiException { + private ApiResponse, Optional>> searchObservationsSearchResultsDbIdGet(UUID programId, String searchResultsDbId, Integer page, Integer pageSize) throws ApiException { ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); return api.searchObservationsSearchResultsDbIdGet(APPLICATION_JSON, searchResultsDbId, page, pageSize); } - public List createBrAPIObservation(List brAPIObservationList, UUID programId, ImportUpload upload) throws ApiException { + public List createBrAPIObservations(List brAPIObservationList, UUID programId, ImportUpload upload) throws ApiException { ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); return brAPIDAOUtil.post(brAPIObservationList, upload, api::observationsPost, importDAO::update); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java index f9a313455..59c2a3041 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java @@ -17,11 +17,13 @@ package org.breedinginsight.brapps.importer.daos; +import io.micronaut.context.annotation.Property; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.request.BrAPIObservationUnitSearchRequest; import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.Program; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; @@ -29,8 +31,8 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.List; -import java.util.UUID; +import javax.validation.constraints.NotNull; +import java.util.*; @Singleton public class BrAPIObservationUnitDAO { @@ -42,12 +44,15 @@ public class BrAPIObservationUnitDAO { private final BrAPIDAOUtil brAPIDAOUtil; private final BrAPIEndpointProvider brAPIEndpointProvider; + private final String referenceSource; + @Inject - public BrAPIObservationUnitDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, BrAPIEndpointProvider brAPIEndpointProvider) { + public BrAPIObservationUnitDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, BrAPIEndpointProvider brAPIEndpointProvider, @Property(name = "brapi.server.reference-source") String referenceSource) { this.programDAO = programDAO; this.importDAO = importDAO; this.brAPIDAOUtil = brAPIDAOUtil; this.brAPIEndpointProvider = brAPIEndpointProvider; + this.referenceSource = referenceSource; } /* @@ -70,6 +75,10 @@ public List getObservationUnitsByNameAndStudyName(List getObservationUnitByName(List observationUnitNames, Program program) throws ApiException { + if(observationUnitNames.isEmpty()) { + return Collections.emptyList(); + } + BrAPIObservationUnitSearchRequest observationUnitSearchRequest = new BrAPIObservationUnitSearchRequest(); observationUnitSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); observationUnitSearchRequest.observationUnitNames(observationUnitNames); @@ -85,4 +94,33 @@ public List createBrAPIObservationUnits(List getObservationUnitsById(Collection observationUnitExternalIds, Program program) throws ApiException { + if(observationUnitExternalIds.isEmpty()) { + return Collections.emptyList(); + } + + BrAPIObservationUnitSearchRequest observationUnitSearchRequest = new BrAPIObservationUnitSearchRequest(); + observationUnitSearchRequest.programDbIds(List.of(program.getBrapiProgram() + .getProgramDbId())); + observationUnitSearchRequest.externalReferenceIDs(new ArrayList<>(observationUnitExternalIds)); + observationUnitSearchRequest.externalReferenceSources(List.of(String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName()))); + + ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationUnitsApi.class); + return brAPIDAOUtil.search(api::searchObservationunitsPost, + api::searchObservationunitsSearchResultsDbIdGet, + observationUnitSearchRequest); + } + + public List getObservationUnitsForStudyDbId(@NotNull String studyDbId, Program program) throws ApiException { + BrAPIObservationUnitSearchRequest observationUnitSearchRequest = new BrAPIObservationUnitSearchRequest(); + observationUnitSearchRequest.programDbIds(List.of(program.getBrapiProgram() + .getProgramDbId())); + observationUnitSearchRequest.studyDbIds(List.of(studyDbId)); + + ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationUnitsApi.class); + return brAPIDAOUtil.search(api::searchObservationunitsPost, + api::searchObservationunitsSearchResultsDbIdGet, + observationUnitSearchRequest); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationVariableDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationVariableDAO.java index 75ebe3502..86731a0f2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationVariableDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationVariableDAO.java @@ -26,6 +26,7 @@ import javax.inject.Inject; import javax.inject.Singleton; +import java.util.Collections; import java.util.List; import java.util.UUID; @@ -44,6 +45,10 @@ public BrAPIObservationVariableDAO(ProgramDAO programDAO, BrAPIDAOUtil brAPIDAOU } public List getVariableByName(List variableNames, UUID programId) throws ApiException { + if(variableNames.isEmpty()) { + return Collections.emptyList(); + } + BrAPIObservationVariableSearchRequest variableSearch = new BrAPIObservationVariableSearchRequest(); variableSearch.observationVariableNames(variableNames); ObservationVariablesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationVariablesApi.class); diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIProgramDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIProgramDAO.java index bd3c7dcca..103585849 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIProgramDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIProgramDAO.java @@ -26,6 +26,7 @@ import org.brapi.v2.model.core.response.BrAPIProgramListResponse; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; @@ -61,10 +62,6 @@ public Optional getProgram(UUID programId) throws ApiException { } List programs = programsResponse.getBody().getResult().getData(); - if (programs.size() == 1) { - return Optional.of(programs.get(0)); - } else { - return Optional.empty(); - } + return Utilities.getSingleOptional(programs); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java index df7965c93..0ca2600b8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java @@ -4,28 +4,16 @@ import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.model.queryParams.core.SeasonQueryParams; -import org.brapi.client.v2.modules.core.ListsApi; import org.brapi.client.v2.modules.core.SeasonsApi; -import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.BrAPIResponse; -import org.brapi.v2.model.BrAPIResponseResult; -import org.brapi.v2.model.core.BrAPIListSummary; -import org.brapi.v2.model.core.BrAPIListTypes; import org.brapi.v2.model.core.BrAPISeason; -import org.brapi.v2.model.core.request.BrAPIListNewRequest; -import org.brapi.v2.model.core.request.BrAPIListSearchRequest; -import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; import org.brapi.v2.model.core.response.BrAPISeasonListResponse; import org.brapi.v2.model.core.response.BrAPISeasonListResponseResult; import org.brapi.v2.model.core.response.BrAPISeasonSingleResponse; -import org.brapi.v2.model.pheno.BrAPIObservation; -import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.utilities.BrAPIDAOUtil; import javax.inject.Inject; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.UUID; @@ -34,30 +22,25 @@ public class BrAPISeasonDAO { private ProgramDAO programDAO; - private ImportDAO importDAO; private final BrAPIEndpointProvider brAPIEndpointProvider; @Inject - public BrAPISeasonDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIEndpointProvider brAPIEndpointProvider) { + public BrAPISeasonDAO(ProgramDAO programDAO, BrAPIEndpointProvider brAPIEndpointProvider) { this.programDAO = programDAO; - this.importDAO = importDAO; this.brAPIEndpointProvider = brAPIEndpointProvider; } - public List getSeasonByYear(String year, UUID programId) throws ApiException { + public List getSeasonsByYear(String year, UUID programId) throws ApiException { SeasonsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), SeasonsApi.class); SeasonQueryParams queryParams = SeasonQueryParams.builder() .year( year ) .pageSize( 10000 ) .build(); - List seasons = new ArrayList<>(); ApiResponse apiResponse = api.seasonsGet( queryParams ); BrAPISeasonListResponse seasonListResponse = apiResponse.getBody(); BrAPISeasonListResponseResult result = seasonListResponse.getResult(); - seasons = result.getData(); - - return seasons; + return result.getData(); } public BrAPISeason getSeasonById(String id, UUID programId) throws ApiException { diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java index f0277fc08..456c7adf5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java @@ -31,8 +31,8 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.List; -import java.util.UUID; +import javax.validation.constraints.NotNull; +import java.util.*; @Singleton public class BrAPIStudyDAO { @@ -52,7 +52,15 @@ public BrAPIStudyDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil br this.brAPIEndpointProvider = brAPIEndpointProvider; } - public List getStudyByName(List studyNames, Program program) throws ApiException { + public Optional getStudyByName(String studyName, Program program) throws ApiException { + List studies = getStudiesByName(List.of(studyName), program); + return Utilities.getSingleOptional(studies); + } + public List getStudiesByName(List studyNames, Program program) throws ApiException { + if(studyNames.isEmpty()) { + return Collections.emptyList(); + } + BrAPIStudySearchRequest studySearch = new BrAPIStudySearchRequest(); studySearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); studySearch.studyNames(studyNames); @@ -64,7 +72,7 @@ public List getStudyByName(List studyNames, Program program) ); } - public List getStudiesByExperimentID(UUID experimentID, Program program ) throws ApiException { + public List getStudiesByExperimentID(@NotNull UUID experimentID, Program program ) throws ApiException { BrAPIStudySearchRequest studySearch = new BrAPIStudySearchRequest(); studySearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); studySearch.addExternalReferenceIDsItem(experimentID.toString()); @@ -77,9 +85,30 @@ public List getStudiesByExperimentID(UUID experimentID, Program prog ); } - public List createBrAPIStudy(List brAPIStudyList, UUID programId, ImportUpload upload) throws ApiException { + public List createBrAPIStudies(List brAPIStudyList, UUID programId, ImportUpload upload) throws ApiException { StudiesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), StudiesApi.class); return brAPIDAOUtil.post(brAPIStudyList, upload, api::studiesPost, importDAO::update); } + public List getStudiesByStudyDbId(Collection studyDbIds, Program program) throws ApiException { + if(studyDbIds.isEmpty()) { + return Collections.emptyList(); + } + + BrAPIStudySearchRequest studySearch = new BrAPIStudySearchRequest(); + studySearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + studySearch.studyDbIds(new ArrayList<>(studyDbIds)); + StudiesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), StudiesApi.class); + return brAPIDAOUtil.search( + api::searchStudiesPost, + api::searchStudiesSearchResultsDbIdGet, + studySearch + ); + } + + public Optional getStudyByDbId(String studyDbId, Program program) throws ApiException { + List studies = getStudiesByStudyDbId(List.of(studyDbId), program); + + return Utilities.getSingleOptional(studies); + } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index fe7d16ad6..44b1b9ba3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -18,11 +18,8 @@ import io.micronaut.context.annotation.Property; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.client.v2.modules.core.StudiesApi; import org.brapi.client.v2.modules.core.TrialsApi; -import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.core.request.BrAPIStudySearchRequest; import org.brapi.v2.model.core.request.BrAPITrialSearchRequest; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; @@ -36,10 +33,7 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.UUID; +import java.util.*; @Singleton public class BrAPITrialDAO { @@ -62,7 +56,11 @@ public BrAPITrialDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil br this.brAPIEndpointProvider = brAPIEndpointProvider; } - public List getTrialByName(List trialNames, Program program) throws ApiException { + public List getTrialsByName(List trialNames, Program program) throws ApiException { + if(trialNames.isEmpty()) { + return Collections.emptyList(); + } + BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); trialSearch.trialNames(trialNames); @@ -74,7 +72,7 @@ public List getTrialByName(List trialNames, Program program) ); } - public List createBrAPITrial(List brAPITrialList, UUID programId, ImportUpload upload) throws ApiException { + public List createBrAPITrials(List brAPITrialList, UUID programId, ImportUpload upload) throws ApiException { TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); return brAPIDAOUtil.post(brAPITrialList, upload, api::trialsPost, importDAO::update); } @@ -91,12 +89,7 @@ public List getTrials(UUID programId) throws ApiException, DoesNotEx trialSearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))); trialSearch.externalReferenceIDs(List.of(programId.toString())); - Optional optionalProgram = programService.getById(programId); - if (!optionalProgram.isPresent()) - { - throw new DoesNotExistException("Program id does not exist"); - } - Program program = optionalProgram.get(); + Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); @@ -124,6 +117,27 @@ private List processExperimentsForDisplay(List trials, S return displayExperiments; } + public Optional getTrialByDbId(String trialDbId, Program program) throws ApiException { + List trials = getTrialsByDbIds(List.of(trialDbId), program); + + return Utilities.getSingleOptional(trials); + } + + public List getTrialsByDbIds(Collection trialDbIds, Program program) throws ApiException { + if(trialDbIds.isEmpty()) { + return Collections.emptyList(); + } + + BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); + trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + trialSearch.trialDbIds(new ArrayList<>(trialDbIds)); + TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); + return brAPIDAOUtil.search( + api::searchTrialsPost, + api::searchTrialsSearchResultsDbIdGet, + trialSearch + ); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/base/Location.java b/src/main/java/org/breedinginsight/brapps/importer/model/base/Location.java index 547147412..fefbe993b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/base/Location.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/base/Location.java @@ -19,11 +19,11 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; -import org.brapi.v2.model.core.BrAPILocation; import org.breedinginsight.brapps.importer.model.config.ImportFieldMetadata; import org.breedinginsight.brapps.importer.model.config.ImportFieldType; import org.breedinginsight.brapps.importer.model.config.ImportFieldTypeEnum; import org.breedinginsight.brapps.importer.model.config.ImportMappingRequired; +import org.breedinginsight.model.ProgramLocation; @Getter @Setter @@ -36,9 +36,9 @@ public class Location implements BrAPIObject { @ImportFieldMetadata(id="locationName", name="Location Name", description = "The name of the location.") private String locationName; - public BrAPILocation constructBrAPILocation() { - BrAPILocation location = new BrAPILocation(); - location.setLocationName(getLocationName()); + public ProgramLocation constructLocation() { + ProgramLocation location = new ProgramLocation(); + location.setName(getLocationName()); return location; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java index 42b4699b2..f3d4f3180 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java @@ -28,6 +28,7 @@ import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.ProgramLocation; import java.util.ArrayList; import java.util.List; @@ -39,7 +40,7 @@ public class PendingImport { private PendingImportObject germplasm; private PendingImportObject trial; - private PendingImportObject location; + private PendingImportObject location; private PendingImportObject study; private PendingImportObject observationUnit; private List> observations = new ArrayList<>(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 19c4a3c22..b12f8ec40 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.breedinginsight.brapps.importer.model.imports.experimentObservation; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 1c50402a2..d429ee126 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -24,12 +24,12 @@ import org.brapi.v2.model.core.*; import org.brapi.v2.model.pheno.*; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapps.importer.model.base.Observation; import org.breedinginsight.brapps.importer.model.config.*; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.model.BrAPIConstants; import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.model.User; import org.breedinginsight.utilities.Utilities; @@ -42,76 +42,76 @@ @Getter @Setter @NoArgsConstructor -@ImportConfigMetadata(id="ExperimentImport", name="Experiment Import", +@ImportConfigMetadata(id = "ExperimentImport", name = "Experiment Import", description = "This import is used to create Observation Unit and Experiment data") public class ExperimentObservation implements BrAPIImport { - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="germplasmName", name="Germplasm Name", description = "Name of germplasm") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "germplasmName", name = Columns.GERMPLASM_NAME, description = "Name of germplasm") private String germplasmName; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="gid", name="Germplasm GID", description = "Unique germplasm identifier") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "gid", name = Columns.GERMPLASM_GID, description = "Unique germplasm identifier") private String gid; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="test_or_check", name="Test or Check", description = "T test (T) and check (C) germplasm") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "test_or_check", name = Columns.TEST_CHECK, description = "T test (T) and check (C) germplasm") private String testOrCheck; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="exp_title", name="Experiment Title", description = "Title of experiment") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "exp_title", name = Columns.EXP_TITLE, description = "Title of experiment") private String expTitle; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="expDescription", name="Experiment Description", description = "Description of experiment") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "expDescription", name = Columns.EXP_DESCRIPTION, description = "Description of experiment") private String expDescription; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="expUnit", name="Experiment Unit", description = "experiment unit (Examples: plots, plant, tanks, hives, etc.)") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "expUnit", name = Columns.EXP_UNIT, description = "experiment unit (Examples: plots, plant, tanks, hives, etc.)") private String expUnit; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="expType", name="Experiment Type", description = "Description of experimental type (Examples: Performance trial, crossing block, seed orchard, etc)") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "expType", name = Columns.EXP_TYPE, description = "Description of experimental type (Examples: Performance trial, crossing block, seed orchard, etc)") private String expType; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="env", name="Environment", description = "Free-text unique identifier for environment within the experiment. Common examples include: 1,2,3…n and/or a concationation of environment location and year") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "env", name = Columns.ENV, description = "Free-text unique identifier for environment within the experiment. Common examples include: 1,2,3…n and/or a concationation of environment location and year") private String env; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="envLocation", name="Environment Location", description = "Location of the environment") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "envLocation", name = Columns.ENV_LOCATION, description = "Location of the environment") private String envLocation; - @ImportFieldType(type= ImportFieldTypeEnum.INTEGER) - @ImportFieldMetadata(id="envYear", name="Environment Year", description = "Year corresponding to the environment") + @ImportFieldType(type = ImportFieldTypeEnum.INTEGER) + @ImportFieldMetadata(id = "envYear", name = Columns.ENV_YEAR, description = "Year corresponding to the environment") private String envYear; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="expUnitId", name="Experiment Unit ID", description = "Human-readable alphanumeric identifier for experimental units unique within environment. Examples, like plot number, are often a numeric sequence.") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "expUnitId", name = Columns.EXP_UNIT_ID, description = "Human-readable alphanumeric identifier for experimental units unique within environment. Examples, like plot number, are often a numeric sequence.") private String expUnitId; - @ImportFieldType(type= ImportFieldTypeEnum.INTEGER) - @ImportFieldMetadata(id="expReplicateNo", name="Experiment Replicate Number", description = "Sequential number of experimental replications") + @ImportFieldType(type = ImportFieldTypeEnum.INTEGER) + @ImportFieldMetadata(id = "expReplicateNo", name = Columns.REP_NUM, description = "Sequential number of experimental replications") private String expReplicateNo; - @ImportFieldType(type= ImportFieldTypeEnum.INTEGER) - @ImportFieldMetadata(id="expBlockNo", name="Experiment Block Number", description = "Sequential number of blocks in an experimental design") + @ImportFieldType(type = ImportFieldTypeEnum.INTEGER) + @ImportFieldMetadata(id = "expBlockNo", name = Columns.BLOCK_NUM, description = "Sequential number of blocks in an experimental design") private String expBlockNo; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="row", name="Row", description = "Horizontal (y-axis) position in 2D Cartesian space.") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "row", name = Columns.ROW, description = "Horizontal (y-axis) position in 2D Cartesian space.") private String row; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="column", name="Column", description = "Vertical (x-axis) position in 2D Cartesian space.") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "column", name = Columns.COLUMN, description = "Vertical (x-axis) position in 2D Cartesian space.") private String column; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="treatmentFactors", name="Treatment Factors", description = "Treatment factors in an experiment with applied variables, like fertilizer or water regimens.") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "treatmentFactors", name = Columns.TREATMENT_FACTORS, description = "Treatment factors in an experiment with applied variables, like fertilizer or water regimens.") private String treatmentFactors; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="obsUnitID", name="Observation Unit ID", description = "A database generated unique identifier for experimental observation units") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "ObsUnitID", name = Columns.OBS_UNIT_ID, description = "A database generated unique identifier for experimental observation units") private String obsUnitID; public BrAPITrial constructBrAPITrial(Program program, User user, boolean commit, String referenceSource, UUID id, String expSeqValue) { @@ -119,9 +119,8 @@ public BrAPITrial constructBrAPITrial(Program program, User user, boolean commit BrAPITrial trial = new BrAPITrial(); if (commit) { setBrAPITrialCommitFields(program, trial, referenceSource, id); - } - else{ - trial.setTrialName( getExpTitle() ); + } else { + trial.setTrialName(getExpTitle()); } trial.setTrialDescription(getExpDescription()); trial.setActive(true); @@ -129,18 +128,20 @@ public BrAPITrial constructBrAPITrial(Program program, User user, boolean commit trial.setProgramName(brapiProgram.getProgramName()); Map createdBy = new HashMap<>(); - createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_ID, user.getId().toString()); + createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_ID, + user.getId() + .toString()); createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_NAME, user.getName()); trial.putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_BY, createdBy); - trial.putAdditionalInfoItem( BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL, getExpUnit()); - trial.putAdditionalInfoItem( BrAPIAdditionalInfoFields.EXPERIMENT_TYPE, getExpType()); - trial.putAdditionalInfoItem( BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER, expSeqValue); + trial.putAdditionalInfoItem(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL, getExpUnit()); + trial.putAdditionalInfoItem(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE, getExpType()); + trial.putAdditionalInfoItem(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER, expSeqValue); return trial; } private void setBrAPITrialCommitFields(Program program, BrAPITrial trial, String referenceSource, UUID id) { - trial.setTrialName( Utilities.appendProgramKey(getExpTitle(), program.getKey() )); + trial.setTrialName(Utilities.appendProgramKey(getExpTitle(), program.getKey())); // Set external reference trial.setExternalReferences(getTrialExternalReferences(program, referenceSource, id)); @@ -151,9 +152,9 @@ private void setBrAPITrialCommitFields(Program program, BrAPITrial trial, String } - public BrAPILocation constructBrAPILocation() { - BrAPILocation location = new BrAPILocation(); - location.setLocationName(getEnvLocation()); + public ProgramLocation constructProgramLocation() { + ProgramLocation location = new ProgramLocation(); + location.setName(getEnvLocation()); return location; } @@ -166,13 +167,12 @@ public BrAPIStudy constructBrAPIStudy( UUID id, Supplier envNextVal) { BrAPIStudy study = new BrAPIStudy(); - if ( commit ){ + if (commit) { study.setStudyName(Utilities.appendProgramKey(getEnv(), program.getKey(), expSequenceValue)); // Set external reference study.setExternalReferences(getStudyExternalReferences(program, referenceSource, trialId, id)); - } - else { + } else { study.setStudyName(getEnv()); } study.setActive(true); @@ -181,20 +181,18 @@ public BrAPIStudy constructBrAPIStudy( study.setTrialName(getExpTitle()); List seasonList = new ArrayList<>(); - seasonList.add( getEnvYear() ); - study.setSeasons( seasonList ); + seasonList.add(getEnvYear()); + study.setSeasons(seasonList); String designType = "Analysis"; // to support the BRApi server, the design type must be one of the following: - // 'CRD','Alpha','MAD','Lattice','Augmented','RCBD','p-rep','splitplot','greenhouse','Westcott', or 'Analysis' - // For now it will be hardcoded to 'Analysis' + // 'CRD','Alpha','MAD','Lattice','Augmented','RCBD','p-rep','splitplot','greenhouse','Westcott', or 'Analysis' + // For now it will be hardcoded to 'Analysis' BrAPIStudyExperimentalDesign design = new BrAPIStudyExperimentalDesign(); design.setPUI(designType); design.setDescription(designType); study.setExperimentalDesign(design); - String envSequenceValue = null; - if( commit ){ - envSequenceValue = envNextVal.get().toString(); - study.putAdditionalInfoItem( BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER, envSequenceValue); + if (commit) { + study.putAdditionalInfoItem(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER, envNextVal.get().toString()); } return study; } @@ -211,18 +209,17 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( ) { BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); - if( commit){ - observationUnit.setObservationUnitName( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal) ); + if (commit) { + observationUnit.setObservationUnitName(Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal)); // Set external reference observationUnit.setExternalReferences(getObsUnitExternalReferences(program, referenceSource, trialID, studyID, id)); - } - else { + } else { observationUnit.setObservationUnitName(getExpUnitId()); } observationUnit.setStudyName(getEnv()); - if(germplasmName==null){ + if (germplasmName == null) { germplasmName = getGermplasmName(); } observationUnit.setGermplasmName(germplasmName); @@ -230,13 +227,13 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); level.setLevelName("plot"); //BreedBase only accepts "plot" or "plant" - level.setLevelCode( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal) ); + level.setLevelCode(Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal)); position.setObservationLevel(level); observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, getExpUnit()); // Exp Unit List levelRelationships = new ArrayList<>(); - if( getExpReplicateNo() !=null ) { + if (getExpReplicateNo() != null) { BrAPIObservationUnitLevelRelationship repLvl = new BrAPIObservationUnitLevelRelationship(); repLvl.setLevelName(BrAPIConstants.REPLICATE.getValue()); repLvl.setLevelCode(getExpReplicateNo()); @@ -244,16 +241,16 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( } // Block number - if( getExpBlockNo() != null ) { - BrAPIObservationUnitLevelRelationship repLvl = new BrAPIObservationUnitLevelRelationship(); - repLvl.setLevelName( BrAPIConstants.BLOCK.getValue() ); - repLvl.setLevelCode(getExpBlockNo()); - levelRelationships.add(repLvl); + if (getExpBlockNo() != null) { + BrAPIObservationUnitLevelRelationship blockLvl = new BrAPIObservationUnitLevelRelationship(); + blockLvl.setLevelName(BrAPIConstants.BLOCK.getValue()); + blockLvl.setLevelCode(getExpBlockNo()); + levelRelationships.add(blockLvl); } position.setObservationLevelRelationships(levelRelationships); // Test or Check - if("C".equals(getTestOrCheck())){ + if ("C".equals(getTestOrCheck())) { position.setEntryType(BrAPIEntryTypeEnum.CHECK); } else { position.setEntryType(BrAPIEntryTypeEnum.TEST); @@ -266,7 +263,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( } if (getColumn() != null) { position.setPositionCoordinateY(getColumn()); - position.setPositionCoordinateYType(BrAPIPositionCoordinateTypeEnum.GRID_ROW); + position.setPositionCoordinateYType(BrAPIPositionCoordinateTypeEnum.GRID_COL); } observationUnit.setObservationUnitPosition(position); @@ -289,14 +286,14 @@ public BrAPIObservation constructBrAPIObservation( String variableName, String seasonDbId, BrAPIObservationUnit obsUnit - ) { + ) { BrAPIObservation observation = new BrAPIObservation(); observation.setGermplasmName(getGermplasmName()); - if(getEnv() != null) { + if (getEnv() != null) { observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.STUDY_NAME, getEnv()); } observation.setObservationVariableName(variableName); - observation.setObservationDbId(obsUnit.getObservationUnitDbId()); + observation.setObservationUnitDbId(obsUnit.getObservationUnitDbId()); observation.setObservationUnitName(obsUnit.getObservationUnitName()); observation.setValue(value); @@ -313,9 +310,15 @@ private List getBrAPIExternalReferences( List refs = new ArrayList<>(); addReference(refs, program.getId(), referenceSourceBaseName, ExternalReferenceSource.PROGRAMS); - if( trialId != null ) { addReference(refs, trialId, referenceSourceBaseName, ExternalReferenceSource.TRIALS); } - if( studyId != null ) { addReference(refs, studyId, referenceSourceBaseName, ExternalReferenceSource.STUDIES); } - if( obsUnitId != null ) { addReference(refs, obsUnitId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATION_UNITS); } + if (trialId != null) { + addReference(refs, trialId, referenceSourceBaseName, ExternalReferenceSource.TRIALS); + } + if (studyId != null) { + addReference(refs, studyId, referenceSourceBaseName, ExternalReferenceSource.STUDIES); + } + if (obsUnitId != null) { + addReference(refs, obsUnitId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATION_UNITS); + } return refs; } @@ -324,22 +327,44 @@ private List getTrialExternalReferences( Program program, String referenceSourceBaseName, UUID trialId) { return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, null, null); } + private List getStudyExternalReferences( Program program, String referenceSourceBaseName, UUID trialId, UUID studyId) { return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, studyId, null); } + private List getObsUnitExternalReferences( Program program, String referenceSourceBaseName, UUID trialId, UUID studyId, UUID obsUnitId) { - return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, studyId, null); + return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, studyId, obsUnitId); } private void addReference(List refs, UUID uuid, String referenceBaseNameSource, ExternalReferenceSource refSourceName) { BrAPIExternalReference reference; reference = new BrAPIExternalReference(); - reference.setReferenceSource( String.format("%s/%s", referenceBaseNameSource, refSourceName.getName()) ); + reference.setReferenceSource(String.format("%s/%s", referenceBaseNameSource, refSourceName.getName())); reference.setReferenceID(uuid.toString()); refs.add(reference); } + public static final class Columns { + public static final String GERMPLASM_NAME = "Germplasm Name"; + public static final String GERMPLASM_GID = "Germplasm GID"; + public static final String TEST_CHECK = "Test (T) or Check (C )"; + public static final String EXP_TITLE = "Exp Title"; + public static final String EXP_DESCRIPTION = "Exp Description"; + public static final String EXP_UNIT = "Exp Unit"; + public static final String EXP_TYPE = "Exp Type"; + public static final String ENV = "Env"; + public static final String ENV_LOCATION = "Env Location"; + public static final String ENV_YEAR = "Env Year"; + public static final String EXP_UNIT_ID = "Exp Unit ID"; + public static final String REP_NUM = "Exp Replicate #"; + public static final String BLOCK_NUM = "Exp Block #"; + public static final String ROW = "Row"; + public static final String COLUMN = "Column"; + public static final String TREATMENT_FACTORS = "Treatment Factors"; + public static final String OBS_UNIT_ID = "ObsUnitID"; + } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 3bd85b6df..19bdbd1cb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -28,6 +28,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.tika.mime.MediaType; import org.brapi.client.v2.JSON; +import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.daos.ImportMappingProgramDAO; @@ -51,6 +52,7 @@ import org.breedinginsight.services.parsers.MimeTypeParser; import org.breedinginsight.services.parsers.ParsingException; import org.breedinginsight.utilities.FileUtil; +import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; import org.jooq.JSONB; import tech.tablesaw.api.Table; @@ -328,9 +330,8 @@ public ImportResponse updateUpload(UUID programId, UUID uploadId, AuthenticatedU User user = userService.getById(actingUser.getId()).get(); // Find the import - Optional uploadOptional = importDAO.getUploadById(uploadId); - if (uploadOptional.isEmpty()) throw new DoesNotExistException("Upload with that id does not exist"); - ImportUpload upload = uploadOptional.get(); + ImportUpload upload = importDAO.getUploadById(uploadId) + .orElseThrow(() -> new DoesNotExistException("Upload with that id does not exist")); if (upload.getProgress() != null && upload.getProgress().getStatuscode().equals((short) HttpStatus.ACCEPTED.getCode())) { // Another action is in process for this import, throw an error @@ -343,13 +344,11 @@ public ImportResponse updateUpload(UUID programId, UUID uploadId, AuthenticatedU } // Get mapping - Optional mappingConfigOptional = importMappingDAO.getMapping(upload.getImporterMappingId()); - if (mappingConfigOptional.isEmpty()) throw new DoesNotExistException("Cannot find mapping config associated with upload."); - ImportMapping mappingConfig = mappingConfigOptional.get(); + ImportMapping mappingConfig = importMappingDAO.getMapping(upload.getImporterMappingId()) + .orElseThrow(() -> new DoesNotExistException("Cannot find mapping config associated with upload.")); - Optional optionalImportService = configManager.getImportServiceById(mappingConfig.getImportTypeId()); - if (optionalImportService.isEmpty()) throw new DoesNotExistException("Config with that id does not exist"); - BrAPIImportService importService = optionalImportService.get(); + BrAPIImportService importService = configManager.getImportServiceById(mappingConfig.getImportTypeId()) + .orElseThrow(() -> new DoesNotExistException("Config with that id does not exist")); // TODO: maybe return brapiimport from configmanager // Get our data @@ -463,7 +462,11 @@ private void processFile(List finalBrAPIImportList, Table data, Pro progress.setUpdatedBy(actingUser.getId()); importDAO.update(upload); } catch (Exception e) { - log.error(e.getMessage(), e); + if(e instanceof ApiException) { + log.error("Error making BrAPI call: " + Utilities.generateApiExceptionLogMessage((ApiException) e), e); + } else { + log.error(e.getMessage(), e); + } ImportProgress progress = upload.getProgress(); progress.setStatuscode((short) HttpStatus.INTERNAL_SERVER_ERROR.getCode()); // TODO: Probably don't want to return this message. But do it for now diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index dda9a417e..3eabe8c4d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -26,32 +26,34 @@ import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.core.BrAPILocation; import org.brapi.v2.model.core.BrAPISeason; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.brapi.v2.model.pheno.*; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.FileMappingUtil; import org.breedinginsight.dao.db.tables.pojos.TraitEntity; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.Scale; -import org.breedinginsight.model.Trait; -import org.breedinginsight.model.User; +import org.breedinginsight.model.*; import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.ValidatorException; @@ -78,34 +80,41 @@ public class ExperimentProcessor implements Processor { private static final String NAME = "Experiment"; - private static final String MISSING_OBS_UNIT_ID_ERROR = "Experiment Units are missing Observation Unit Id.\n" + + private static final String MISSING_OBS_UNIT_ID_ERROR = "Experiment Units are missing Observation Unit Id.

" + "If you’re trying to add these units to the experiment, please create a new environment" + " with all appropriate experiment units (NOTE: this will generate new Observation Unit Ids " + "for each experiment unit)."; + private static final String MIDNIGHT = "T00:00:00-00:00"; + private static final String TIMESTAMP_PREFIX = "TS:"; + private static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; + private static final String COMMA_DELIMITER = ","; + private static final String BLANK_FIELD_EXPERIMENT = "Field is blank when creating a new experiment"; + private static final String BLANK_FIELD_ENV = "Field is blank when creating a new environment"; + private static final String BLANK_FIELD_OBS = "Field is blank when creating new observations"; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; - private DSLContext dsl; - private BrAPITrialDAO brapiTrialDAO; - private BrAPILocationDAO brAPILocationDAO; - private BrAPIStudyDAO brAPIStudyDAO; - private BrAPIObservationUnitDAO brAPIObservationUnitDAO; - private BrAPIObservationDAO brAPIObservationDAO; - private BrAPISeasonDAO brAPISeasonDAO; - private BrAPIGermplasmDAO brAPIGermplasmDAO; - private OntologyService ontologyService; - private FileMappingUtil fileMappingUtil; + private final DSLContext dsl; + private final BrAPITrialDAO brapiTrialDAO; + private final ProgramLocationService locationService; + private final BrAPIStudyDAO brAPIStudyDAO; + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private final BrAPIObservationDAO brAPIObservationDAO; + private final BrAPISeasonDAO brAPISeasonDAO; + private final BrAPIGermplasmDAO brAPIGermplasmDAO; + private final OntologyService ontologyService; + private final FileMappingUtil fileMappingUtil; // used to make the yearsToSeasonDbId() function more efficient - private final Map yearToSeasonDbIdCache = new HashMap<>(); + private final Map yearToSeasonDbIdCache = new HashMap<>(); // used to make the seasonDbIdtoYear() function more efficient - private final Map seasonDbIdToYearCache = new HashMap<>(); + private final Map seasonDbIdToYearCache = new HashMap<>(); //These BrapiData-objects are initially populated by the getExistingBrapiData() method, // then updated by the getNewBrapiData() method. private Map> trialByNameNoScope = null; - private Map> locationByName = null; + private Map> locationByName = null; private Map> studyByNameNoScope = null; // It is assumed that there are no preexisting Observation Units for the given environment (so this will not be // initialized by getExistingBrapiData() ) @@ -117,12 +126,12 @@ public class ExperimentProcessor implements Processor { private Map> existingGermplasmByGID = null; // Associates timestamp columns to associated phenotype column name for ease of storage - private Map timeStampColByPheno = new HashMap<>(); + private Map> timeStampColByPheno = new HashMap<>(); @Inject public ExperimentProcessor(DSLContext dsl, BrAPITrialDAO brapiTrialDAO, - BrAPILocationDAO brAPILocationDAO, + ProgramLocationService locationService, BrAPIStudyDAO brAPIStudyDAO, BrAPIObservationUnitDAO brAPIObservationUnitDAO, BrAPIObservationDAO brAPIObservationDAO, @@ -132,7 +141,7 @@ public ExperimentProcessor(DSLContext dsl, FileMappingUtil fileMappingUtil) { this.dsl = dsl; this.brapiTrialDAO = brapiTrialDAO; - this.brAPILocationDAO = brAPILocationDAO; + this.locationService = locationService; this.brAPIStudyDAO = brAPIStudyDAO; this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.brAPIObservationDAO = brAPIObservationDAO; @@ -142,8 +151,14 @@ public ExperimentProcessor(DSLContext dsl, this.fileMappingUtil = fileMappingUtil; } + @Override + public String getName() { + return NAME; + } + /** * Initialize the Map objects with existing BrAPI Data. + * * @param importRows * @param program */ @@ -151,26 +166,23 @@ public ExperimentProcessor(DSLContext dsl, public void getExistingBrapiData(List importRows, Program program) { List experimentImportRows = importRows.stream() - .map(trialImport -> (ExperimentObservation) trialImport) - .collect(Collectors.toList()); - - this.trialByNameNoScope = initialize_trialByNameNoScope( program, experimentImportRows ); - this.locationByName = initialize_uniqueLocationNames( program, experimentImportRows ); - this.studyByNameNoScope = initialize_studyByNameNoScope( program, experimentImportRows ); - // All of the Observation Units will be new. None will be preexisting. - this.observationUnitByNameNoScope = new HashMap<>(); - // TODO: populate existing observations, assume all new currently - // key and removing key - this.existingGermplasmByGID = initialize_existingGermplasmByGID( program, experimentImportRows ); + .map(trialImport -> (ExperimentObservation) trialImport) + .collect(Collectors.toList()); + + this.observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); + this.trialByNameNoScope = initializeTrialByNameNoScope(program, experimentImportRows); + this.studyByNameNoScope = initializeStudyByNameNoScope(program, experimentImportRows); + this.locationByName = initializeUniqueLocationNames(program, experimentImportRows); + this.existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); } /** - * @param importRows - one element of the list for every row of the import file. + * @param importRows - one element of the list for every row of the import file. * @param mappedBrAPIImport - passed in by reference and modified within this program (this will later be passed to the front end for the preview) * @param program * @param user - * @param commit - true when the data should be saved (ie when the user has pressed the "Commit" button) - * false when used for preview only + * @param commit - true when the data should be saved (ie when the user has pressed the "Commit" button) + * false when used for preview only * @return Map - used to display the summary statistics. * @throws ValidatorException */ @@ -181,7 +193,8 @@ public Map process( Table data, Program program, User user, - boolean commit) throws ValidatorException, MissingRequiredInfoException { + boolean commit) throws ValidatorException, MissingRequiredInfoException, ApiException { + log.debug("processing experiment import"); ValidationErrors validationErrors = new ValidationErrors(); @@ -189,84 +202,129 @@ public Map process( List> dynamicCols = fileMappingUtil.getDynamicColumns(data, EXPERIMENT_TEMPLATE_NAME); List> phenotypeCols = new ArrayList<>(); List> timestampCols = new ArrayList<>(); - for (Column dynamicCol: dynamicCols) { + for (Column dynamicCol : dynamicCols) { //Distinguish between phenotype and timestamp columns - if (dynamicCol.name().startsWith("TS:")) { + if (dynamicCol.name().startsWith(TIMESTAMP_PREFIX)) { timestampCols.add(dynamicCol); } else { phenotypeCols.add(dynamicCol); } } - List varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toList()); - List tsNames = timestampCols.stream().map(Column::name).collect(Collectors.toList()); + List referencedTraits = verifyTraits(program.getId(), phenotypeCols, timestampCols, validationErrors); - // Lookup all traits in system for program, maybe eventually add a variable search in ontology service - List traits = getTraitList(program); + //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval + for (Column tsColumn : timestampCols) { + timeStampColByPheno.put(tsColumn.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), tsColumn); + } - // filter out just traits specified in file - List filteredTraits = traits.stream() - .filter(e -> varNames.contains(e.getObservationVariableName())) - .collect(Collectors.toList()); + // add "New" pending data to the BrapiData objects + initNewBrapiData(importRows, phenotypeCols, program, user, commit); - // check that all specified ontology terms were found - if (filteredTraits.size() != varNames.size()) { - List returnedVarNames = filteredTraits.stream().map(TraitEntity::getObservationVariableName) - .collect(Collectors.toList()); - List differences = varNames.stream() - .filter(var -> !returnedVarNames.contains(var)) - .collect(Collectors.toList()); - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, - "Ontology term(s) not found: " + String.join(", ", differences)); - } + prepareDataForValidation(importRows, phenotypeCols, mappedBrAPIImport); - // Check that each ts column corresponds to a phenotype column - List unmatchedTimestamps = tsNames.stream() - .filter(e -> !(varNames.contains(e.replaceFirst("^TS:\\s*","")))) - .collect(Collectors.toList()); - if (unmatchedTimestamps.size() > 0) { - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, - "Timestamp column(s) lack corresponding phenotype column(s): " + String.join(", ", unmatchedTimestamps)); - } + validateFields(importRows, validationErrors, mappedBrAPIImport, referencedTraits, program, phenotypeCols); - //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval - for (Column tsColumn: timestampCols) { - timeStampColByPheno.put(tsColumn.name().replaceFirst("^TS:\\s*",""), tsColumn); + if (validationErrors.hasErrors()) { + throw new ValidatorException(validationErrors); } - // Perform ontology validations on each observation value in phenotype column - Map colVarMap = filteredTraits.stream() - .collect(Collectors.toMap(Trait::getObservationVariableName, Function.identity())); + log.debug("done processing experiment import"); + // Construct our response object + return generateStatisticsMap(importRows); + } - for (Column column : phenotypeCols) { - for (int i=0; i < column.size(); i++) { - String value = column.getString(i); - String colName = column.name(); - validateObservationValue(colVarMap.get(colName), value, colName, validationErrors, i); + @Override + public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { + // TODO + } + + @Override + public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) { + log.debug("starting post of experiment data to BrAPI server"); + + List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); + List newLocations = ProcessorData.getNewObjects(this.locationByName) + .stream() + .map(location -> ProgramLocationRequest.builder() + .name(location.getName()) + .build()) + .collect(Collectors.toList()); + List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); + List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); + // filter out observations with no 'value' so they will not be saved + List newObservations = ProcessorData.getNewObjects(this.observationByHash) + .stream() + .filter(obs -> !obs.getValue().isBlank()) + .collect(Collectors.toList()); + + AuthenticatedUser actingUser = new AuthenticatedUser(upload.getUpdatedByUser().getName(), new ArrayList<>(), upload.getUpdatedByUser().getId(), new ArrayList<>()); + + try { + List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newTrials, program.getId(), upload)); + // set the DbId to the for each newly created trial + for (BrAPITrial createdTrial : createdTrials) { + String createdTrialName = Utilities.removeProgramKey(createdTrial.getTrialName(), program.getKey()); + this.trialByNameNoScope.get(createdTrialName) + .getBrAPIObject() + .setTrialDbId(createdTrial.getTrialDbId()); } - } - //Timestamp validation - for (Column column : timestampCols) { - for (int i=0; i < column.size(); i++) { - String value = column.getString(i); - String colName = column.name(); - validateTimeStampValue(value, colName, validationErrors, i); + List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); + // set the DbId to the for each newly created trial + for (ProgramLocation createdLocation : createdLocations) { + String createdLocationName = createdLocation.getName(); + this.locationByName.get(createdLocationName) + .getBrAPIObject() + .setLocationDbId(createdLocation.getLocationDbId()); } - } - // add "New" pending data to the BrapiData objects - getNewBrapiData(importRows, phenotypeCols, program, user, commit); + updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); + List createdStudies = brAPIStudyDAO.createBrAPIStudies(newStudies, program.getId(), upload); + + // set the DbId to the for each newly created study + for (BrAPIStudy createdStudy : createdStudies) { + String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData(createdStudy.getStudyName(), program.getKey()); + this.studyByNameNoScope.get(createdStudy_name_no_key) + .getBrAPIObject() + .setStudyDbId(createdStudy.getStudyDbId()); + } + + updateObsUnitDependencyValues(program.getKey()); + List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); + + // set the DbId to the for each newly created Observation Unit + for (BrAPIObservationUnit createdObservationUnit : createdObservationUnits) { + // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope + String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getStudyName(), program.getKey()); + String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getObservationUnitName(), program.getKey()); + String createdObsUnit_key = createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); + this.observationUnitByNameNoScope.get(createdObsUnit_key) + .getBrAPIObject() + .setObservationUnitDbId(createdObservationUnit.getObservationUnitDbId()); + } + + updateObservationDependencyValues(program); + brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); + log.debug("experiment import complete"); + } catch (ApiException e) { + log.error("Error saving experiment import: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error saving experiment import", e); + throw new InternalServerException(e.getMessage(), e); + } + } - // For each import row - for (int i = 0; i < importRows.size(); i++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); + private void prepareDataForValidation(List importRows, List> phenotypeCols, Map mappedBrAPIImport) { + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(i, new PendingImport()); - mappedImportRow.setTrial( this.trialByNameNoScope.get( importRow.getExpTitle() ) ); - mappedImportRow.setLocation( this.locationByName.get( importRow.getEnvLocation() ) ); - mappedImportRow.setStudy( this.studyByNameNoScope.get( importRow.getEnv() ) ); - mappedImportRow.setObservationUnit( this.observationUnitByNameNoScope.get( createObservationUnitKey( importRow ) ) ); + PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); + mappedImportRow.setTrial(this.trialByNameNoScope.get(importRow.getExpTitle())); + mappedImportRow.setLocation(this.locationByName.get(importRow.getEnvLocation())); + mappedImportRow.setStudy(this.studyByNameNoScope.get(importRow.getEnv())); + mappedImportRow.setObservationUnit(this.observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); // loop over phenotype column observation data for current row for (Column column : phenotypeCols) { @@ -277,38 +335,68 @@ public Map process( } PendingImportObject germplasmPIO = getGidPOI(importRow); - mappedImportRow.setGermplasm( germplasmPIO ); + mappedImportRow.setGermplasm(germplasmPIO); - if (! StringUtils.isBlank( importRow.getGid() )) { // if GID is blank, don't bother to check if it is valid. - validateGermplasm(importRow,validationErrors, i, germplasmPIO); - } + mappedBrAPIImport.put(rowNum, mappedImportRow); + } + } - //Check if existing environment. If so, ObsUnitId must be assigned - if ((this.studyByNameNoScope.get(importRow.getEnv()).getState() == ImportObjectState.EXISTING) && (StringUtils.isBlank(importRow.getObsUnitID()))){ - throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); - } + private List verifyTraits(UUID programId, List> phenotypeCols, List> timestampCols, ValidationErrors validationErrors) { + Set varNames = phenotypeCols.stream() + .map(Column::name) + .collect(Collectors.toSet()); + Set tsNames = timestampCols.stream() + .map(Column::name) + .collect(Collectors.toSet()); - mappedBrAPIImport.put(i, mappedImportRow); - } - // End-of-loop + // filter out just traits specified in file + List filteredTraits = fetchFileTraits(programId, varNames); - validationErrors = validateFields(importRows, validationErrors); + // check that all specified ontology terms were found + if (filteredTraits.size() != varNames.size()) { + Set returnedVarNames = filteredTraits.stream() + .map(TraitEntity::getObservationVariableName) + .collect(Collectors.toSet()); + List differences = varNames.stream() + .filter(var -> !returnedVarNames.contains(var)) + .collect(Collectors.toList()); + //TODO convert this to a ValidationError + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, + "Ontology term(s) not found: " + String.join(COMMA_DELIMITER, differences)); + } - if (validationErrors.hasErrors()){ - throw new ValidatorException(validationErrors); + // Check that each ts column corresponds to a phenotype column + List unmatchedTimestamps = tsNames.stream() + .filter(e -> !(varNames.contains(e.replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY)))) + .collect(Collectors.toList()); + if (unmatchedTimestamps.size() > 0) { + //TODO convert this to a ValidationError + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, + "Timestamp column(s) lack corresponding phenotype column(s): " + String.join(COMMA_DELIMITER, unmatchedTimestamps)); } - // Construct our response object - return getStatisticsMap(importRows); + return filteredTraits; } + private List fetchFileTraits(UUID programId, Collection varNames) { + try { + List traits = ontologyService.getTraitsByProgramId(programId, true); + // filter out just traits specified in file + return traits.stream() + .filter(e -> varNames.contains(e.getObservationVariableName())) + .collect(Collectors.toList()); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + throw new InternalServerException(e.toString(), e); + } + } private String getVariableNameFromColumn(Column column) { // TODO: timestamp stripping? return column.name(); } - private void getNewBrapiData(List importRows, List> phenotypeCols, Program program, User user, boolean commit) { + private void initNewBrapiData(List importRows, List> phenotypeCols, Program program, User user, boolean commit) { String expSequenceName = program.getExpSequence(); if (expSequenceName == null) { @@ -324,59 +412,56 @@ private void getNewBrapiData(List importRows, List> pheno } Supplier envNextVal = () -> dsl.nextval(envSequenceName.toLowerCase()); - for (int i=0; i trialPIO = createTrialPIO(program, user, commit, importRow, expNextVal); - this.trialByNameNoScope.put(importRow.getExpTitle(), trialPIO); + PendingImportObject trialPIO = fetchOrCreateTrialPIO(program, user, commit, importRow, expNextVal); String expSeqValue = null; - if(commit) { - expSeqValue = trialPIO.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString(); + if (commit) { + expSeqValue = trialPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString(); } - PendingImportObject locationPIO = createLocationPIO(importRow); - this.locationByName.put(importRow.getEnvLocation(), locationPIO); + fetchOrCreateLocationPIO(importRow); - PendingImportObject studyPIO = createStudyPIO(program, commit, expSeqValue, importRow, envNextVal); - this.studyByNameNoScope.put(importRow.getEnv(), studyPIO); + PendingImportObject studyPIO = fetchOrCreateStudyPIO(program, commit, expSeqValue, importRow, envNextVal); String envSeqValue = null; - if(commit) { - envSeqValue = studyPIO.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER).getAsString(); + if (commit) { + envSeqValue = studyPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER) + .getAsString(); } - PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, envSeqValue, importRow); - String key = createObservationUnitKey(importRow); - this.observationUnitByNameNoScope.put(key, obsUnitPIO); + PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); for (Column column : phenotypeCols) { //If associated timestamp column, add String dateTimeValue = null; - if (timeStampColByPheno.get(column.name()) != null) { - dateTimeValue = timeStampColByPheno.get(column.name()).getString(i); + if (timeStampColByPheno.containsKey(column.name())) { + dateTimeValue = timeStampColByPheno.get(column.name()).getString(rowNum); //If no timestamp, set to midnight - if (!dateTimeValue.isBlank() && !validDateTimeValue(dateTimeValue)){ - dateTimeValue+="T00:00:00-00:00"; + if (!dateTimeValue.isBlank() && !validDateTimeValue(dateTimeValue)) { + dateTimeValue += MIDNIGHT; } } //column.name() gets phenotype name String seasonDbId = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); - PendingImportObject obsPIO = createObservationPIO(importRow, column.name(), column.getString(i), dateTimeValue, commit, seasonDbId, obsUnitPIO); - this.observationByHash.put(getImportObservationHash(importRow, getVariableNameFromColumn(column)), obsPIO); + fetchOrCreateObservationPIO(importRow, column.name(), column.getString(rowNum), dateTimeValue, commit, seasonDbId, obsUnitPIO); } } } private String createObservationUnitKey(ExperimentObservation importRow) { - String key = createObservationUnitKey( importRow.getEnv(), importRow.getExpUnitId() ); - return key; + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); } private String createObservationUnitKey(String studyName, String obsUnitName) { - String key = studyName + obsUnitName; - return key; + return studyName + obsUnitName; } private String getImportObservationHash(ExperimentObservation importRow, String variableName) { @@ -390,106 +475,176 @@ private String getObservationHash(String observationUnitName, String variableNam return DigestUtils.sha256Hex(concat); } - private ValidationErrors validateFields(List importRows, ValidationErrors validationErrors) { - HashSet uniqueStudyAndObsUnit = new HashSet<>(); - for (int i = 0; i < importRows.size(); i++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); - validateConditionallyRequired(validationErrors, i, importRow); - validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, i, importRow); + private ValidationErrors validateFields(List importRows, ValidationErrors validationErrors, Map mappedBrAPIImport, List referencedTraits, Program program, + List> phenotypeCols) throws MissingRequiredInfoException, ApiException { + //fetching any existing observations for any OUs in the import + Map existingObsByObsHash = fetchExistingObservations(referencedTraits, program); + Map colVarMap = referencedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableName, Function.identity())); + + Set uniqueStudyAndObsUnit = new HashSet<>(); + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); + + if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. + validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); + } + + //Check if existing environment. If so, ObsUnitId must be assigned + if ((mappedImportRow.getStudy().getState() == ImportObjectState.EXISTING) + && (StringUtils.isBlank(importRow.getObsUnitID()))) { + throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); + } + + validateConditionallyRequired(validationErrors, rowNum, importRow); + validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, existingObsByObsHash); } return validationErrors; } + private void validateObservationUnits(ValidationErrors validationErrors, Set uniqueStudyAndObsUnit, int rowNum, ExperimentObservation importRow) { + validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + + String key = createObservationUnitKey(importRow); + PendingImportObject ouPIO = observationUnitByNameNoScope.get(key); + if(ouPIO.getState() == ImportObjectState.NEW && StringUtils.isNotBlank(importRow.getObsUnitID())) { + addRowError(Columns.OBS_UNIT_ID, "Could not find observation unit by ObsUnitDBID", validationErrors, rowNum); + } + } + + private Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { + Set ouDbIds = new HashSet<>(); + Set variableDbIds = new HashSet<>(); + Map variableNameByDbId = new HashMap<>(); + Map ouNameByDbId = new HashMap<>(); + Map studyNameByDbId = studyByNameNoScope.values() + .stream() + .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) + .map(PendingImportObject::getBrAPIObject) + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); + + observationUnitByNameNoScope.values().forEach(ou -> { + if(StringUtils.isNotBlank(ou.getBrAPIObject().getObservationUnitDbId())) { + ouDbIds.add(ou.getBrAPIObject().getObservationUnitDbId()); + } + ouNameByDbId.put(ou.getBrAPIObject().getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getBrAPIObject().getObservationUnitName(), program.getKey())); + }); + + for (Trait referencedTrait : referencedTraits) { + variableDbIds.add(referencedTrait.getObservationVariableDbId()); + variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); + } + + List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); + + return existingObservations.stream() + .map(obs -> { + String studyName = studyNameByDbId.get(obs.getStudyDbId()); + String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); + String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); + + String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); + + return Map.entry(key, obs); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private void validateObservations(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, List> phenotypeCols, Map colVarMap, Map existingObservations) { + phenotypeCols.forEach(phenoCol -> { + if(existingObservations.containsKey(getImportObservationHash(importRow, phenoCol.name()))) { + addRowError( + phenoCol.name(), + String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), + validationErrors, rowNum + ); + } else { + validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); + + //Timestamp validation + if(timeStampColByPheno.containsKey(phenoCol.name())) { + Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); + validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); + } + } + }); + } + /** - * Validate that the the observation unit is unique within a study. + * Validate that the observation unit is unique within a study. *
* SIDE EFFECTS: validationErrors and uniqueStudyAndObsUnit can be modified. - * @param validationErrors can be modified as a side effect. + * + * @param validationErrors can be modified as a side effect. * @param uniqueStudyAndObsUnit can be modified as a side effect. - * @param i counter that is always two less the file row being validated - * @param importRow the data row being validated + * @param rowNum counter that is always two less the file row being validated + * @param importRow the data row being validated */ private void validateUniqueObsUnits( ValidationErrors validationErrors, - HashSet uniqueStudyAndObsUnit, - int i, + Set uniqueStudyAndObsUnit, + int rowNum, ExperimentObservation importRow) { - String envIdPlusStudyId = createObservationUnitKey( importRow ); - if( uniqueStudyAndObsUnit.contains( envIdPlusStudyId )){ + String envIdPlusStudyId = createObservationUnitKey(importRow); + if (uniqueStudyAndObsUnit.contains(envIdPlusStudyId)) { String errorMessage = String.format("The ID (%s) is not unique within the environment(%s)", importRow.getExpUnitId(), importRow.getEnv()); - this.addRowError("Exp Unit ID", errorMessage, validationErrors, i); - } - else{ - uniqueStudyAndObsUnit.add( envIdPlusStudyId ); + this.addRowError(Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); + } else { + uniqueStudyAndObsUnit.add(envIdPlusStudyId); } } - private void validateConditionallyRequired(ValidationErrors validationErrors, int i, ExperimentObservation importRow) { - String experimentTitle = importRow.getExpTitle(); - String obsUnitID = importRow.getObsUnitID(); - if( StringUtils.isBlank( obsUnitID )){ - validateRequiredCell( - experimentTitle, - "Exp Title", - "Field is blank", validationErrors, i - ); + private void validateConditionallyRequired(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow) { + ImportObjectState expState = this.trialByNameNoScope.get(importRow.getExpTitle()) + .getState(); + ImportObjectState envState = this.studyByNameNoScope.get(importRow.getEnv()).getState(); + + String errorMessage = BLANK_FIELD_EXPERIMENT; + if (expState == ImportObjectState.EXISTING && envState == ImportObjectState.NEW) { + errorMessage = BLANK_FIELD_ENV; + } else if(expState == ImportObjectState.EXISTING && envState == ImportObjectState.EXISTING) { + errorMessage = BLANK_FIELD_OBS; } - ImportObjectState expState = this.trialByNameNoScope.get(experimentTitle).getState(); - boolean isExperimentNew = (expState == ImportObjectState.NEW); - - if (isExperimentNew) { - String errorMessage = "Field is blank when creating a new experiment"; - validateRequiredCell( importRow.getGid(), - "GID", - errorMessage, validationErrors, i); - validateRequiredCell( importRow.getExpUnit(), - "Exp Unit", - errorMessage, validationErrors, i); - validateRequiredCell( importRow.getExpType(), - "Exp Type", - errorMessage, validationErrors, i); - validateRequiredCell( importRow.getEnv(), - "Env", - errorMessage, validationErrors, i); - validateRequiredCell( importRow.getEnvLocation(), - "Env Location", - errorMessage, validationErrors, i); - validateRequiredCell( importRow.getEnvYear(), - "Env Year", - errorMessage, validationErrors, i); - validateRequiredCell( importRow.getExpUnitId(), - "Exp Unit ID", - errorMessage, validationErrors, i); - validateRequiredCell( importRow.getExpReplicateNo(), - "Exp Replicate #", - errorMessage, validationErrors, i); - validateRequiredCell( importRow.getExpBlockNo(), - "Exp Block #", - errorMessage, validationErrors, i); + if(expState == ImportObjectState.NEW || envState == ImportObjectState.NEW) { + validateRequiredCell(importRow.getGid(), Columns.GERMPLASM_GID, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpTitle(),Columns.EXP_TITLE,errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpUnit(), Columns.EXP_UNIT, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpType(), Columns.EXP_TYPE, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getEnv(), Columns.ENV, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getEnvLocation(), Columns.ENV_LOCATION, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getEnvYear(), Columns.ENV_YEAR, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpUnitId(), Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpReplicateNo(), Columns.REP_NUM, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpBlockNo(), Columns.BLOCK_NUM, errorMessage, validationErrors, rowNum); + + if(StringUtils.isNotBlank(importRow.getObsUnitID())) { + addRowError(Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); + } + } else { + validateRequiredCell(importRow.getObsUnitID(), Columns.OBS_UNIT_ID, errorMessage, validationErrors, rowNum); } } - private void validateRequiredCell(String value, String columnHeader, String errorMessage, ValidationErrors validationErrors, int i) { - if ( StringUtils.isBlank( value )) { - addRowError( - columnHeader, - errorMessage, - validationErrors, i - ) ; + private void validateRequiredCell(String value, String columnHeader, String errorMessage, ValidationErrors validationErrors, int rowNum) { + if (StringUtils.isBlank(value)) { + addRowError(columnHeader, errorMessage, validationErrors, rowNum); } } - private void addRowError(String field, String errorMessage, ValidationErrors validationErrors, int i) { + private void addRowError(String field, String errorMessage, ValidationErrors validationErrors, int rowNum) { ValidationError ve = new ValidationError(field, errorMessage, HttpStatus.UNPROCESSABLE_ENTITY); - validationErrors.addError(i + 2, ve); // +2 instead of +1 to account for the column header row. + validationErrors.addError(rowNum + 2, ve); // +2 instead of +1 to account for the column header row. } private void addIfNotNull(HashSet set, String setValue) { - if( setValue!=null) { set.add(setValue); } + if (setValue != null) { + set.add(setValue); + } } - private Map getStatisticsMap(List importRows) { + private Map generateStatisticsMap(List importRows) { // Data for stats. HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's @@ -499,86 +654,92 @@ private Map getStatisticsMap(List ExperimentObservation importRow = (ExperimentObservation) row; // Collect date for stats. addIfNotNull(environmentNameCounter, importRow.getEnv()); - addIfNotNull(obsUnitsIDCounter, createObservationUnitKey( importRow )); + addIfNotNull(obsUnitsIDCounter, createObservationUnitKey(importRow)); addIfNotNull(gidCounter, importRow.getGid()); } int numNewObservations = Math.toIntExact( - observationByHash.values().stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && - !StringUtils.isBlank(preview.getBrAPIObject().getValue())) - .count() + observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() ); ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() - .newObjectCount(environmentNameCounter.size()) - .build(); + .newObjectCount(environmentNameCounter.size()) + .build(); ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() - .newObjectCount(obsUnitsIDCounter.size()) - .build(); + .newObjectCount(obsUnitsIDCounter.size()) + .build(); ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() - .newObjectCount(gidCounter.size()) - .build(); + .newObjectCount(gidCounter.size()) + .build(); ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder() - .newObjectCount(numNewObservations) - .build(); + .newObjectCount(numNewObservations) + .build(); return Map.of( - "Environments", environmentStats, - "Observation_Units", obdUnitStats, - "GIDs", gidStats, - "Observations", observationStats + "Environments", environmentStats, + "Observation_Units", obdUnitStats, + "GIDs", gidStats, + "Observations", observationStats ); } - private void validateGermplasm(ExperimentObservation importRow, ValidationErrors validationErrors, int i, PendingImportObject germplasmPIO) { - // error if GID is not blank but GID does not already exist - if( !StringUtils.isBlank( importRow.getGid()) && germplasmPIO == null ) { - addRowError( - "GID", - "A non-existing GID", - validationErrors, i - ); + private void validateGermplasm(ExperimentObservation importRow, ValidationErrors validationErrors, int rowNum, PendingImportObject germplasmPIO) { + // error if GID is not blank but GID does not already exist + if (StringUtils.isNotBlank(importRow.getGid()) && germplasmPIO == null) { + addRowError(Columns.GERMPLASM_GID, "A non-existing GID", validationErrors, rowNum); } } private PendingImportObject getGidPOI(ExperimentObservation importRow) { - if( this.existingGermplasmByGID.containsKey( importRow.getGid() )){ + if (this.existingGermplasmByGID.containsKey(importRow.getGid())) { return existingGermplasmByGID.get(importRow.getGid()); } - else{ - return null; - } + + return null; } - private PendingImportObject createObsUnitPIO(Program program, boolean commit, String seqValue, ExperimentObservation importRow) { - PendingImportObject pio = null; - if( this.observationUnitByNameNoScope.containsKey( createObservationUnitKey( importRow ) ) ) { - pio = observationUnitByNameNoScope.get( createObservationUnitKey( importRow ) ) ; - } - else{ + private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) { + PendingImportObject pio; + String key = createObservationUnitKey(importRow); + if (this.observationUnitByNameNoScope.containsKey(key)) { + pio = observationUnitByNameNoScope.get(key); + } else { String germplasmName = ""; - if( this.existingGermplasmByGID.get( importRow.getGid() ) != null) { - germplasmName = this.existingGermplasmByGID.get(importRow.getGid()).getBrAPIObject().getGermplasmName(); + if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { + germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) + .getBrAPIObject() + .getGermplasmName(); } PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, seqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, studyID, id); - pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, studyID, id); + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); + this.observationUnitByNameNoScope.put(key, pio); } return pio; } - private PendingImportObject createObservationPIO(ExperimentObservation importRow, String variableName, String value, String timeStampValue, boolean commit, String seasonDbId, PendingImportObject obsUnitPIO) { - PendingImportObject pio = null; - if (this.observationByHash.containsKey(getImportObservationHash(importRow, variableName))) { - pio = observationByHash.get(getImportObservationHash(importRow, variableName)); - } - else { + private PendingImportObject fetchOrCreateObservationPIO(ExperimentObservation importRow, + String variableName, + String value, + String timeStampValue, + boolean commit, + String seasonDbId, + PendingImportObject obsUnitPIO) { + PendingImportObject pio; + String key = getImportObservationHash(importRow, variableName); + if (this.observationByHash.containsKey(key)) { + pio = observationByHash.get(key); + } else { BrAPIObservation newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject()); //NOTE: Can't parse invalid timestamp value, so have to skip if invalid. // Validation error should be thrown for offending value, but that doesn't happen until later downstream @@ -586,265 +747,205 @@ private PendingImportObject createObservationPIO(ExperimentObs newObservation.setObservationTimeStamp(OffsetDateTime.parse(timeStampValue)); } + newObservation.setStudyDbId(this.studyByNameNoScope.get(importRow.getEnv()).getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); + this.observationByHash.put(key, pio); } return pio; } - private PendingImportObject createStudyPIO(Program program, boolean commit, String expSequenceValue, ExperimentObservation importRow, Supplier envNextVal) { - PendingImportObject pio = null; - if( studyByNameNoScope.containsKey( importRow.getEnv()) ) { - pio = studyByNameNoScope.get( importRow.getEnv() ) ; - } - else{ + private PendingImportObject fetchOrCreateStudyPIO(Program program, boolean commit, String expSequenceValue, ExperimentObservation importRow, Supplier envNextVal) { + PendingImportObject pio; + if (studyByNameNoScope.containsKey(importRow.getEnv())) { + pio = studyByNameNoScope.get(importRow.getEnv()); + } else { PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); UUID id = UUID.randomUUID(); BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); + newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies - if( commit) { + if (commit) { String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season - String seasonID = this.yearToSeasonDbId(year, program.getId()); - newStudy.setSeasons(Arrays.asList(seasonID)); + if(StringUtils.isNotBlank(year)) { + String seasonID = this.yearToSeasonDbId(year, program.getId()); + newStudy.setSeasons(Collections.singletonList(seasonID)); + } } pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); + this.studyByNameNoScope.put(importRow.getEnv(), pio); } return pio; } - private PendingImportObject createLocationPIO(ExperimentObservation importRow) { - PendingImportObject pio = null; - if( locationByName.containsKey(( importRow.getEnvLocation() ))){ - pio = locationByName.get( importRow.getEnvLocation() ); - } - else{ - BrAPILocation newLocation = importRow.constructBrAPILocation(); - pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation); + private PendingImportObject fetchOrCreateLocationPIO(ExperimentObservation importRow) { + PendingImportObject pio; + if (locationByName.containsKey((importRow.getEnvLocation()))) { + pio = locationByName.get(importRow.getEnvLocation()); + } else { + ProgramLocation newLocation = importRow.constructProgramLocation(); + pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); + this.locationByName.put(importRow.getEnvLocation(), pio); } return pio; } - private PendingImportObject createTrialPIO(Program program, User user, boolean commit, ExperimentObservation importRow, Supplier expNextVal) { - PendingImportObject pio = null; - if( trialByNameNoScope.containsKey( importRow.getExpTitle()) ) { - pio = trialByNameNoScope.get( importRow.getExpTitle() ) ; - } - else { + private PendingImportObject fetchOrCreateTrialPIO(Program program, User user, boolean commit, ExperimentObservation importRow, Supplier expNextVal) { + PendingImportObject pio; + if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { + pio = trialByNameNoScope.get(importRow.getExpTitle()); + } else { UUID id = UUID.randomUUID(); String expSeqValue = null; - if(commit){ + if (commit) { expSeqValue = expNextVal.get().toString(); } BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); pio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); + this.trialByNameNoScope.put(importRow.getExpTitle(), pio); } return pio; } - @Override - public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { - // TODO - } - - @Override - public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) { - - List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); - List newLocations = ProcessorData.getNewObjects(this.locationByName); - List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); - List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); - // filter out observations with no 'value' so they will not be saved - List newObservations = ProcessorData.getNewObjects(this.observationByHash).stream() - .filter(obs -> !obs.getValue().isBlank()) - .collect(Collectors.toList()); - try { - List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrial(newTrials, program.getId(), upload)); - // set the DbId to the for each newly created trial - for ( BrAPITrial createdTrial: createdTrials ) { - String createdTrialName = createdTrial.getTrialName(); - String createdTrialName_no_key = Utilities.removeProgramKey( createdTrialName, program.getKey() ); - PendingImportObject pi = this.trialByNameNoScope.get(createdTrialName_no_key); - BrAPITrial listedTrial = pi.getBrAPIObject(); - String dbid = createdTrial.getTrialDbId(); - listedTrial.setTrialDbId( dbid ); - } - - List createdLocations = new ArrayList<>(brAPILocationDAO.createBrAPILocation(newLocations, program.getId(), upload)); - // set the DbId to the for each newly created trial - for ( BrAPILocation createdLocation : createdLocations){ - String createdLocationName = createdLocation.getLocationName(); - PendingImportObject pi = this.locationByName.get(createdLocationName); - BrAPILocation listedLocation = pi.getBrAPIObject(); - String dbid = createdLocation.getLocationDbId(); - listedLocation.setLocationDbId(dbid); - } - - updateStudyDependencyValues(mappedBrAPIImport,program.getKey()); - List createdStudies = new ArrayList<>(); - createdStudies.addAll( brAPIStudyDAO.createBrAPIStudy(newStudies, program.getId(), upload) ); - - // set the DbId to the for each newly created study - for( BrAPIStudy createdStudy : createdStudies){ - String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData( createdStudy.getStudyName(), program.getKey() ); - PendingImportObject pi = this.studyByNameNoScope.get( createdStudy_name_no_key ); - BrAPIStudy brAPIStudy = pi.getBrAPIObject(); - brAPIStudy.setStudyDbId( createdStudy.getStudyDbId() ); - } - - updateObsUnitDependencyValues(program.getKey()); - List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); - - // set the DbId to the for each newly created Observation Unit - for( BrAPIObservationUnit createdObservationUnit : createdObservationUnits){ - // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope - String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData( createdObservationUnit.getStudyName(),program.getKey() ); - String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( createdObservationUnit.getObservationUnitName(),program.getKey() ); - String createdObsUnit_key = createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); - PendingImportObject pi = this.observationUnitByNameNoScope.get(createdObsUnit_key); - // update the retrieved BrAPI object - BrAPIObservationUnit brAPIObservationUnit = pi.getBrAPIObject(); - brAPIObservationUnit.setObservationUnitDbId ( createdObservationUnit.getObservationUnitDbId() ); - } - - updateObservationDependencyValues(program); - brAPIObservationDAO.createBrAPIObservation(newObservations, program.getId(), upload); - } catch (ApiException e) { - log.error(e.getResponseBody()); - throw new InternalServerException(e.toString(), e); - } - } - private void updateObservationDependencyValues(Program program) { String programKey = program.getKey(); // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds this.observationUnitByNameNoScope.values().stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); + .map(PendingImportObject::getBrAPIObject) + .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); // Update ObservationVariable DbIds List traits = getTraitList(program); - Map traitMap = traits.stream().collect(Collectors.toMap(trait -> trait.getObservationVariableName(), trait -> trait)); + Map traitMap = traits.stream().collect(Collectors.toMap(TraitEntity::getObservationVariableName, Function.identity())); - for(PendingImportObject observation: this.observationByHash.values()){ + for (PendingImportObject observation : this.observationByHash.values()) { String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); - if( observationVariableName!=null && traitMap.containsKey(observationVariableName)){ + if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); - observation.getBrAPIObject().setObservationVariableDbId( observationVariableDbId ); + observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); } } } private List getTraitList(Program program) { - List traits = null; try { - traits = ontologyService.getTraitsByProgramId(program.getId(), true); + return ontologyService.getTraitsByProgramId(program.getId(), true); } catch (DoesNotExistException e) { log.error(e.getMessage(), e); throw new InternalServerException(e.toString(), e); } - return traits; } // Update each ovservation's observationUnit DbId, study DbId, and germplasm DbId private void updateObservationDbIds(BrAPIObservationUnit obsUnit, String programKey) { // FILTER LOGIC: Match on Env and Exp Unit ID - this.observationByHash.values().stream() - .filter(obs -> - obs.getBrAPIObject().getAdditionalInfo() != null - && obs.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null - && obs.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.STUDY_NAME) - .getAsString() - .equals(obsUnit.getStudyName()) - && Utilities.removeProgramKeyAndUnknownAdditionalData( - obs.getBrAPIObject().getObservationUnitName(), programKey) - .equals( - Utilities.removeProgramKeyAndUnknownAdditionalData( - obsUnit.getObservationUnitName(), programKey) - ) - ) - .forEach(obs -> { - if(StringUtils.isBlank(obs.getBrAPIObject().getObservationUnitDbId())) { - obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); - } - obs.getBrAPIObject().setStudyDbId(obsUnit.getStudyDbId()); - obs.getBrAPIObject().setGermplasmDbId(obsUnit.getGermplasmDbId()); - }); + this.observationByHash.values() + .stream() + .filter(obs -> obs.getBrAPIObject() + .getAdditionalInfo() != null + && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null + && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) + .getAsString() + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getStudyName(), programKey)) + && Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getBrAPIObject().getObservationUnitName(), programKey) + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getObservationUnitName(), programKey)) + ) + .forEach(obs -> { + if (StringUtils.isBlank(obs.getBrAPIObject().getObservationUnitDbId())) { + obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); + } + obs.getBrAPIObject().setStudyDbId(obsUnit.getStudyDbId()); + obs.getBrAPIObject().setGermplasmDbId(obsUnit.getGermplasmDbId()); + }); } private void updateObsUnitDependencyValues(String programKey) { // update study DbIds - this.studyByNameNoScope.values().stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(study -> updateStudyDbId(study, programKey)); + this.studyByNameNoScope.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(study -> updateStudyDbId(study, programKey)); // update germplasm DbIds - this.existingGermplasmByGID.values().stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(this::updateGermplasmDbId); + this.existingGermplasmByGID.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(this::updateGermplasmDbId); } private void updateStudyDbId(BrAPIStudy study, String programKey) { - this.observationUnitByNameNoScope.values().stream() - .filter(obsUnit -> obsUnit.getBrAPIObject().getStudyName().equals( Utilities.removeProgramKeyAndUnknownAdditionalData( study.getStudyName(), programKey ) )) - .forEach(obsUnit -> { - obsUnit.getBrAPIObject().setStudyDbId(study.getStudyDbId()); - obsUnit.getBrAPIObject().setTrialDbId(study.getTrialDbId()); - }); + this.observationUnitByNameNoScope.values() + .stream() + .filter(obsUnit -> obsUnit.getBrAPIObject() + .getStudyName() + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), programKey))) + .forEach(obsUnit -> { + obsUnit.getBrAPIObject() + .setStudyDbId(study.getStudyDbId()); + obsUnit.getBrAPIObject() + .setTrialDbId(study.getTrialDbId()); + }); } private void updateGermplasmDbId(BrAPIGermplasm germplasm) { - this.observationUnitByNameNoScope.values().stream() - .filter(obsUnit -> obsUnit.getBrAPIObject().getGermplasmName() != null && - obsUnit.getBrAPIObject().getGermplasmName().equals(germplasm.getGermplasmName())) - .forEach(obsUnit -> obsUnit.getBrAPIObject().setGermplasmDbId(germplasm.getGermplasmDbId())); + this.observationUnitByNameNoScope.values() + .stream() + .filter(obsUnit -> obsUnit.getBrAPIObject() + .getGermplasmName() != null && + obsUnit.getBrAPIObject() + .getGermplasmName() + .equals(germplasm.getGermplasmName())) + .forEach(obsUnit -> obsUnit.getBrAPIObject() + .setGermplasmDbId(germplasm.getGermplasmDbId())); } private void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { // update location DbIds in studies for all distinct locations - mappedBrAPIImport.values().stream() - .map(PendingImport::getLocation) - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(this::updateStudyLocationDbId); + mappedBrAPIImport.values() + .stream() + .map(PendingImport::getLocation) + .forEach(this::updateStudyLocationDbId); // update trial DbIds in studies for all distinct trials - this.trialByNameNoScope.values().stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(trait -> this.updateTrialDbId(trait, programKey)); + this.trialByNameNoScope.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(trial -> this.updateTrialDbId(trial, programKey)); } - private void updateStudyLocationDbId(BrAPILocation location) { - this.studyByNameNoScope.values().stream() - .filter(study -> location.getLocationName().equals( study.getBrAPIObject().getLocationName() )) - .forEach(study -> study.getBrAPIObject().setLocationDbId(location.getLocationDbId())); + private void updateStudyLocationDbId(PendingImportObject location) { + this.studyByNameNoScope.values() + .stream() + .filter(study -> location.getId().toString() + .equals(study.getBrAPIObject() + .getLocationDbId())) + .forEach(study -> study.getBrAPIObject() + .setLocationDbId(location.getBrAPIObject().getLocationDbId())); } private void updateTrialDbId(BrAPITrial trial, String programKey) { - this.studyByNameNoScope.values().stream() - .filter(study -> study.getBrAPIObject().getTrialName().equals(Utilities.removeProgramKey(trial.getTrialName(), programKey ) ) ) - .forEach(study -> study.getBrAPIObject().setTrialDbId(trial.getTrialDbId())); - } - - @Override - public String getName() { - return NAME; + this.studyByNameNoScope.values() + .stream() + .filter(study -> study.getBrAPIObject() + .getTrialName() + .equals(Utilities.removeProgramKey(trial.getTrialName(), programKey))) + .forEach(study -> study.getBrAPIObject() + .setTrialDbId(trial.getTrialDbId())); } private ArrayList getGermplasmByAccessionNumber( @@ -853,9 +954,10 @@ private ArrayList getGermplasmByAccessionNumber( List germplasmList = brAPIGermplasmDAO.getGermplasm(programId); ArrayList resultGermplasm = new ArrayList<>(); // Search for accession number matches - for (BrAPIGermplasm germplasm: germplasmList) { - for (String accessionNumber: germplasmAccessionNumbers) { - if (germplasm.getAccessionNumber().equals(accessionNumber)) { + for (BrAPIGermplasm germplasm : germplasmList) { + for (String accessionNumber : germplasmAccessionNumbers) { + if (germplasm.getAccessionNumber() + .equals(accessionNumber)) { resultGermplasm.add(germplasm); break; } @@ -864,118 +966,259 @@ private ArrayList getGermplasmByAccessionNumber( return resultGermplasm; } - private Map> initialize_existingGermplasmByGID(Program program, List experimentImportRows) { - Map> existingGermplasmByGID = new HashMap<>(); - List uniqueGermplasmGIDs = experimentImportRows.stream() - .map(ExperimentObservation::getGid) - .distinct() - .collect(Collectors.toList()); + private Map> initializeObservationUnits(Program program, List experimentImportRows) { + Map> ret = new HashMap<>(); + + Map rowByObsUnitId = new HashMap<>(); + experimentImportRows.forEach(row -> { + if (StringUtils.isNotBlank(row.getObsUnitID())) { + if(rowByObsUnitId.containsKey(row.getObsUnitID())) { + throw new IllegalStateException("ObsUnitId is repeated: " + row.getObsUnitID()); + } + rowByObsUnitId.put(row.getObsUnitID(), row); + } + }); - List existingGermplasms; try { - existingGermplasms = this.getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId()); - existingGermplasms.forEach(existingGermplasm -> existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm))); - return existingGermplasmByGID; + List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); + + String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + if (existingObsUnits.size() == rowByObsUnitId.size()) { + existingObsUnits.forEach(brAPIObservationUnit -> { + BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) + .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); + ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceID()); + row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); + ret.put(createObservationUnitKey(row), + new PendingImportObject<>(ImportObjectState.EXISTING, + brAPIObservationUnit, + UUID.fromString(idRef.getReferenceID()))); + }); + } + + return ret; } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); } } - private Map> initialize_studyByNameNoScope(Program program, List experimentImportRows) { - Map> studyByNameNoScope = new HashMap<>(); - if( this.trialByNameNoScope.size()!=1){ - return studyByNameNoScope; + private Map> initializeTrialByNameNoScope(Program program, List experimentImportRows) { + Map> trialByName = new HashMap<>(); + String programKey = program.getKey(); + + initializeTrialsForExistingObservationUnits(program, trialByName); + + List uniqueTrialNames = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID())) + .map(experimentImport -> Utilities.appendProgramKey(experimentImport.getExpTitle(), programKey)) + .distinct() + .collect(Collectors.toList()); + try { + String trialRefSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName()); + brapiTrialDAO.getTrialsByName(uniqueTrialNames, program) + .forEach(existingTrial -> processAndCacheTrial(existingTrial, program, trialRefSource, trialByName)); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); } - ExperimentObservation experimentObservation = experimentImportRows.get(0); - PendingImportObject trial = this.trialByNameNoScope.get(experimentObservation.getExpTitle()); + return trialByName; + } - UUID experimentId = trial.getId(); + private Map> initializeStudyByNameNoScope(Program program, List experimentImportRows) { + Map> studyByName = new HashMap<>(); + if (this.trialByNameNoScope.size() != 1) { + return studyByName; + } List existingStudies; + Optional expTitle = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID())) + .map(ExperimentObservation::getExpTitle) + .findFirst(); + if(expTitle.isEmpty()) { + expTitle = trialByNameNoScope.keySet().stream().findFirst(); + } + PendingImportObject trial = this.trialByNameNoScope.get(expTitle.get()); + UUID experimentId = trial.getId(); + try { existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); - existingStudies.forEach(existingStudy -> { - //Swap season DbId with year String - String seasonId = existingStudy.getSeasons().get(0); - existingStudy.setSeasons( List.of( this.seasonDbIdToYear( seasonId, program.getId() ) ) ); - - existingStudy.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey())); - studyByNameNoScope.put( - existingStudy.getStudyName(), - new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy)); - }); - return studyByNameNoScope; } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); } + + existingStudies.forEach(existingStudy -> processAndCacheStudy(existingStudy, program, studyByName)); + return studyByName; } - private Map> initialize_uniqueLocationNames(Program program, List experimentImportRows) { - Map> locationByName = new HashMap<>(); + private Map> initializeUniqueLocationNames(Program program, List experimentImportRows) { + Map> locationByName = new HashMap<>(); + + List existingLocations = new ArrayList<>(); + if(studyByNameNoScope.size() > 0) { + Set locationDbIds = studyByNameNoScope.values() + .stream() + .map(study -> study.getBrAPIObject() + .getLocationDbId()) + .collect(Collectors.toSet()); + try { + existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + List uniqueLocationNames = experimentImportRows.stream() - .map(ExperimentObservation::getEnvLocation) - .distinct() - .filter(Objects::nonNull) - .collect(Collectors.toList()); + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getEnvLocation) + .distinct() + .filter(Objects::nonNull) + .collect(Collectors.toList()); - List existingLocations; try { - existingLocations = brAPILocationDAO.getLocationsByName(uniqueLocationNames, program.getId()); - existingLocations.forEach(existingLocation -> { - locationByName.put(existingLocation.getLocationName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation)); - }); - return locationByName; + existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); } + + existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); + return locationByName; } - private Map> initialize_trialByNameNoScope(Program program, List experimentImportRows) { - Map> trialByNameNoScope = new HashMap<>(); - String programKey = program.getKey(); - List uniqueTrialNames = experimentImportRows.stream() - .map(experimentImport -> Utilities.appendProgramKey( experimentImport.getExpTitle(), programKey, null) ) - .distinct() - .filter(Objects::nonNull) - .collect(Collectors.toList()); - List existingTrials; + private Map> initializeExistingGermplasmByGID(Program program, List experimentImportRows) { + Map> existingGermplasmByGID = new HashMap<>(); - try { - existingTrials = brapiTrialDAO.getTrialByName(uniqueTrialNames, program); - existingTrials.forEach(existingTrial -> { - existingTrial.setTrialName(Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey())); - - //get TrialId from existingTrial - List experimentRefs = existingTrial.getExternalReferences(); - Optional experimentIDRef = experimentRefs.stream() - .filter(this::isTrialRefSource) - .findFirst(); - if( experimentIDRef.isEmpty()){ - throw new InternalServerException("An Experiment ID was not found any of the external references"); - } - UUID experimentId = UUID.fromString( experimentIDRef.get().getReferenceID() ); + List existingGermplasms = new ArrayList<>(); + if(observationUnitByNameNoScope.size() > 0) { + Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + try { + existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueGermplasmGIDs = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getGid) + .distinct() + .collect(Collectors.toList()); - trialByNameNoScope.put( - existingTrial.getTrialName(), - new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); - }); - return trialByNameNoScope; + try { + existingGermplasms.addAll(this.getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); } + + existingGermplasms.forEach(existingGermplasm -> { + BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceID()))); + }); + return existingGermplasmByGID; + } + + private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map> studyByName) { + //Swap season DbId with year String + String seasonId = existingStudy.getSeasons().get(0); + existingStudy.setSeasons(List.of(this.seasonDbIdToYear(seasonId, program.getId()))); + + existingStudy.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey())); + + BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); + studyByName.put( + existingStudy.getStudyName(), + new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy, UUID.fromString(xref.getReferenceID()))); + } + + private void initializeTrialsForExistingObservationUnits(Program program, Map> trialByName) { + if(observationUnitByNameNoScope.size() > 0) { + Set trialDbIds = new HashSet<>(); + Set studyDbIds = new HashSet<>(); + + observationUnitByNameNoScope.values() + .forEach(pio -> { + BrAPIObservationUnit existingOu = pio.getBrAPIObject(); + if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { + throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); + } + + if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { + trialDbIds.add(existingOu.getTrialDbId()); + } else { + studyDbIds.add(existingOu.getStudyDbId()); + } + }); + + //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId + if(!studyDbIds.isEmpty()) { + try { + trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, program)); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + try { + List trials = brapiTrialDAO.getTrialsByDbIds(trialDbIds, program); + if (trials.size() != trialDbIds.size()) { + List missingIds = new ArrayList<>(trialDbIds); + missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); + throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(COMMA_DELIMITER, missingIds)); + } + + String trialRefSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName()); + trials.forEach(trial -> processAndCacheTrial(trial, program, trialRefSource, trialByName)); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + } + + private Set fetchTrialDbidsForStudies(Set studyDbIds, Program program) throws ApiException { + Set trialDbIds = new HashSet<>(); + List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); + if(studies.size() != studyDbIds.size()) { + List missingIds = new ArrayList<>(trialDbIds); + missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); + throw new IllegalStateException("Study not found for studyDbId(s): " + String.join(COMMA_DELIMITER, missingIds)); + } + studies.forEach(study -> { + if (StringUtils.isBlank(study.getTrialDbId())) { + throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); + } + trialDbIds.add(study.getTrialDbId()); + }); + + return trialDbIds; } - private String simpleStudyName(String scopedName){ - return scopedName.replaceFirst(" \\[.*\\]", ""); + + private void processAndCacheTrial(BrAPITrial existingTrial, Program program, String trialRefSource, Map> trialByNameNoScope) { + existingTrial.setTrialName(Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey())); + + //get TrialId from existingTrial + BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), trialRefSource) + .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); + UUID experimentId = UUID.fromString(experimentIDRef.getReferenceID()); + + trialByNameNoScope.put( + existingTrial.getTrialName(), + new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); } private void validateTimeStampValue(String value, - String columnHeader, ValidationErrors validationErrors, int row){ - if(StringUtils.isBlank(value)) { + String columnHeader, ValidationErrors validationErrors, int row) { + if (StringUtils.isBlank(value)) { log.debug(String.format("skipping validation of observation timestamp because there is no value.\n\tvariable: %s\n\trow: %d", columnHeader, row)); return; } @@ -984,20 +1227,20 @@ private void validateTimeStampValue(String value, } } + private void validateObservationValue(Trait variable, String value, String columnHeader, ValidationErrors validationErrors, int row) { - if(StringUtils.isBlank(value)) { + if (StringUtils.isBlank(value)) { log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s\n\trow: %d", variable.getObservationVariableName(), row)); return; } - switch(variable.getScale().getDataType()) { + switch (variable.getScale().getDataType()) { case NUMERICAL: Optional number = validNumericValue(value); if (number.isEmpty()) { addRowError(columnHeader, "Non-numeric text detected detected", validationErrors, row); - } - else if (!validNumericRange(number.get(), variable.getScale())) { + } else if (!validNumericRange(number.get(), variable.getScale())) { addRowError(columnHeader, "Value outside of min/max range detected", validationErrors, row); } break; @@ -1021,6 +1264,7 @@ else if (!validNumericRange(number.get(), variable.getScale())) { } } + private Optional validNumericValue(String value) { BigDecimal number; try { @@ -1034,7 +1278,7 @@ private Optional validNumericValue(String value) { private boolean validNumericRange(BigDecimal value, Scale validValues) { // account for empty min or max in valid determination return (validValues.getValidValueMin() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMin())) >= 0) && - (validValues.getValidValueMax() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMax())) <= 0); + (validValues.getValidValueMax() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMax())) <= 0); } private boolean validDateValue(String value) { @@ -1058,7 +1302,9 @@ private boolean validDateTimeValue(String value) { } private boolean validCategory(List categories, String value) { - Set categoryValues = categories.stream().map(category -> category.getValue().toLowerCase()).collect(Collectors.toSet()); + Set categoryValues = categories.stream() + .map(category -> category.getValue().toLowerCase()) + .collect(Collectors.toSet()); return categoryValues.contains(value.toLowerCase()); } @@ -1068,30 +1314,27 @@ private boolean validCategory(List categories, * NOTE: This assumes that the only Season records of interest are ones * with a blank name or a name that is the same as the year. * - * @param year The year as a string + * @param year The year as a string * @param programId the program ID. * @return the DbId of the season-record associated with the year */ private String yearToSeasonDbId(String year, UUID programId) { String dbID = null; - if (this.yearToSeasonDbIdCache.containsKey(year) ){ // get it from cache if possible + if (this.yearToSeasonDbIdCache.containsKey(year)) { // get it from cache if possible dbID = this.yearToSeasonDbIdCache.get(year); - } - else{ - dbID = this.yearToSeasonDbIdFromDatabase(year,programId); + } else { + dbID = this.yearToSeasonDbIdFromDatabase(year, programId); this.yearToSeasonDbIdCache.put(year, dbID); } return dbID; } - private String seasonDbIdToYear(String seasonDbId, UUID programId) { String year = null; - if (this.seasonDbIdToYearCache.containsKey(seasonDbId) ){ // get it from cache if possible + if (this.seasonDbIdToYearCache.containsKey(seasonDbId)) { // get it from cache if possible year = this.seasonDbIdToYearCache.get(seasonDbId); - } - else{ - year = this.seasonDbIdToYearFromDatabase(seasonDbId,programId); + } else { + year = this.seasonDbIdToYearFromDatabase(seasonDbId, programId); this.seasonDbIdToYearCache.put(seasonDbId, year); } return year; @@ -1101,18 +1344,18 @@ private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { BrAPISeason targetSeason = null; List seasons; try { - seasons = this.brAPISeasonDAO.getSeasonByYear(year, programId); - for( BrAPISeason season : seasons){ - if(null == season.getSeasonName() || season.getSeasonName().isBlank() || season.getSeasonName().equals(year)){ + seasons = this.brAPISeasonDAO.getSeasonsByYear(year, programId); + for (BrAPISeason season : seasons) { + if (null == season.getSeasonName() || season.getSeasonName().isBlank() || season.getSeasonName().equals(year)) { targetSeason = season; break; } } - if (targetSeason == null){ + if (targetSeason == null) { BrAPISeason newSeason = new BrAPISeason(); - newSeason.setYear( Integer.parseInt(year) ); - newSeason.setSeasonName( year ); - targetSeason = this.brAPISeasonDAO.addOneSeason( newSeason, programId ); + newSeason.setYear(Integer.parseInt(year)); + newSeason.setSeasonName(year); + targetSeason = this.brAPISeasonDAO.addOneSeason(newSeason, programId); } } catch (ApiException e) { @@ -1120,24 +1363,17 @@ private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { log.error(e.getResponseBody(), e); } - String seasonDbId = (targetSeason==null) ? null : targetSeason.getSeasonDbId(); - return seasonDbId; + return (targetSeason == null) ? null : targetSeason.getSeasonDbId(); } private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { - BrAPISeason targetSeason = null; BrAPISeason season = null; try { - season = this.brAPISeasonDAO.getSeasonById (seasonDbId, programId); + season = this.brAPISeasonDAO.getSeasonById(seasonDbId, programId); } catch (ApiException e) { - log.error(e.getResponseBody(), e); + log.error(Utilities.generateApiExceptionLogMessage(e), e); } Integer yearInt = (season == null) ? null : season.getYear(); - String yearStr = (yearInt==null) ? "" : yearInt.toString(); - return yearStr; - } - - private boolean isTrialRefSource(BrAPIExternalReference brAPIExternalReference) { - return brAPIExternalReference.getReferenceSource().equals( Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS)); + return (yearInt == null) ? "" : yearInt.toString(); } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index 5a2709b28..7276a36ac 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -92,7 +92,6 @@ public class GermplasmProcessor implements Processor { @Inject public GermplasmProcessor(BrAPIGermplasmService brAPIGermplasmService, DSLContext dsl, BreedingMethodDAO breedingMethodDAO, BrAPIListDAO brAPIListDAO) { - this.brAPIGermplasmService = brAPIGermplasmService; this.dsl = dsl; this.breedingMethodDAO = breedingMethodDAO; this.brAPIListDAO = brAPIListDAO; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java index 820fba4ab..cd2debb1e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java @@ -18,9 +18,10 @@ import io.micronaut.context.annotation.Prototype; import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPILocation; -import org.breedinginsight.brapps.importer.daos.BrAPILocationDAO; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.base.Location; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; @@ -29,7 +30,9 @@ import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.model.User; +import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.services.exceptions.ValidatorException; import tech.tablesaw.api.Table; @@ -41,16 +44,17 @@ import java.util.stream.Collectors; @Prototype +@Slf4j public class LocationProcessor implements Processor { private static final String NAME = "Location"; - private BrAPILocationDAO brAPILocationDAO; - private Map> locationByName = new HashMap<>(); + private ProgramLocationService locationService; + private Map> locationByName = new HashMap<>(); @Inject - public LocationProcessor(BrAPILocationDAO brAPILocationDAO) { - this.brAPILocationDAO = brAPILocationDAO; + public LocationProcessor(ProgramLocationService locationService) { + this.locationService = locationService; } public void getExistingBrapiData(List importRows, Program program) { @@ -59,12 +63,12 @@ public void getExistingBrapiData(List importRows, Program program) .map(locationImport -> locationImport.getLocation().getLocationName()) .distinct() .collect(Collectors.toList()); - List existingLocations; + List existingLocations; try { - existingLocations = brAPILocationDAO.getLocationsByName(uniqueLocationNames, program.getId()); + existingLocations = locationService.getLocationsByName(uniqueLocationNames, program.getId()); existingLocations.forEach(existingLocation -> { - locationByName.put(existingLocation.getLocationName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation)); + locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation)); }); } catch (ApiException e) { // We shouldn't get an error back from our services. If we do, nothing the user can do about it @@ -84,9 +88,9 @@ public Map process(List importRows Location location = brapiImport.getLocation(); - BrAPILocation brapiLocation = location.constructBrAPILocation(); + ProgramLocation brapiLocation = location.constructLocation(); if (!locationByName.containsKey(location.getLocationName())) { - locationByName.put(brapiLocation.getLocationName(), new PendingImportObject<>(ImportObjectState.NEW, brapiLocation)); + locationByName.put(brapiLocation.getName(), new PendingImportObject<>(ImportObjectState.NEW, brapiLocation)); mappedImportRow.setLocation(new PendingImportObject<>(ImportObjectState.NEW, brapiLocation)); } mappedImportRow.setLocation(locationByName.get(location.getLocationName())); @@ -108,18 +112,25 @@ public void validateDependencies(Map mappedBrAPIImport) @Override public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) throws ValidatorException { - List locations = ProcessorData.getNewObjects(locationByName); - - List createdLocations = new ArrayList<>(); + List locations = ProcessorData.getNewObjects(this.locationByName) + .stream() + .map(location -> ProgramLocationRequest.builder() + .name(location.getName()) + .build()) + .collect(Collectors.toList()); + + AuthenticatedUser actingUser = new AuthenticatedUser(upload.getUpdatedByUser().getName(), new ArrayList<>(), upload.getUpdatedByUser().getId(), new ArrayList<>()); + List createdLocations = new ArrayList<>(); try { - createdLocations.addAll(brAPILocationDAO.createBrAPILocation(locations, program.getId(), upload)); - } catch (ApiException e) { - throw new InternalServerException(e.toString(), e); + createdLocations.addAll(locationService.create(actingUser, program.getId(), locations)); + } catch (Exception e) { + log.error("Error saving location import", e); + throw new InternalServerException(e.getMessage(), e); } // Update our records createdLocations.forEach(location -> { - PendingImportObject preview = locationByName.get(location.getLocationName()); + PendingImportObject preview = locationByName.get(location.getName()); preview.setBrAPIObject(location); }); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java index 83cf4d2ae..5a8f0c907 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java @@ -192,7 +192,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program List createdObservations = new ArrayList<>(); try { - createdObservations.addAll(brAPIObservationDAO.createBrAPIObservation(observations, program.getId(), upload)); + createdObservations.addAll(brAPIObservationDAO.createBrAPIObservations(observations, program.getId(), upload)); } catch (ApiException e) { throw new InternalServerException(e.toString(), e); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java index 9c72ea4f4..26ba7a1ac 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java @@ -53,7 +53,7 @@ public interface Processor { Map process(List importRows, Map mappedBrAPIImport, Table data, Program program, User user, boolean commit) - throws ValidatorException, MissingRequiredInfoException; + throws ValidatorException, MissingRequiredInfoException, ApiException; /** * Given mapped brapi import with updates from prior dependencies, check if have everything needed diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java index 4a041cad9..8576b01fa 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java @@ -33,6 +33,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.model.User; import org.breedinginsight.services.exceptions.ValidatorException; import tech.tablesaw.api.Table; @@ -67,7 +68,7 @@ public void getExistingBrapiData(List importRows, Program program) List existingStudies; try { - existingStudies = brAPIStudyDAO.getStudyByName(uniqueStudyNames, program); + existingStudies = brAPIStudyDAO.getStudiesByName(uniqueStudyNames, program); existingStudies.forEach(existingStudy -> { studyByName.put(existingStudy.getStudyName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy)); }); @@ -127,7 +128,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program // POST Study List createdStudies = new ArrayList<>(); try { - createdStudies.addAll(brAPIStudyDAO.createBrAPIStudy(studies, program.getId(), upload)); + createdStudies.addAll(brAPIStudyDAO.createBrAPIStudies(studies, program.getId(), upload)); } catch (ApiException e) { throw new InternalServerException(e.toString(), e); } @@ -158,9 +159,9 @@ private void updateDependencyValues(Map mappedBrAPIImpor .forEach(this::updateTrialDbId); } - private void updateLocationDbId(BrAPILocation location) { + private void updateLocationDbId(ProgramLocation location) { this.studyByName.values().stream() - .filter(study -> study.getBrAPIObject().getLocationName().equals(location.getLocationName())) + .filter(study -> study.getBrAPIObject().getLocationName().equals(location.getName())) .forEach(study -> study.getBrAPIObject().setLocationDbId(location.getLocationDbId())); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java index 18ee6d37c..71fa6d04c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java @@ -62,7 +62,7 @@ public void getExistingBrapiData(List importRows, Program program) List existingTrials; try { - existingTrials = brapiTrialDAO.getTrialByName(uniqueTrialNames, program); + existingTrials = brapiTrialDAO.getTrialsByName(uniqueTrialNames, program); existingTrials.forEach(existingTrial -> { trialByName.put(existingTrial.getTrialName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial)); }); @@ -114,7 +114,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program List createdTrials = new ArrayList<>(); try { - createdTrials.addAll(brapiTrialDAO.createBrAPITrial(trials, program.getId(), upload)); + createdTrials.addAll(brapiTrialDAO.createBrAPITrials(trials, program.getId(), upload)); } catch (ApiException e) { throw new InternalServerException(e.toString(), e); } diff --git a/src/main/java/org/breedinginsight/daos/ObservationDAO.java b/src/main/java/org/breedinginsight/daos/ObservationDAO.java index d15a49c28..9ecfb0588 100644 --- a/src/main/java/org/breedinginsight/daos/ObservationDAO.java +++ b/src/main/java/org/breedinginsight/daos/ObservationDAO.java @@ -36,8 +36,8 @@ import javax.inject.Inject; import javax.inject.Singleton; +import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.UUID; import static org.brapi.v2.model.BrAPIWSMIMEDataTypes.APPLICATION_JSON; @@ -76,6 +76,9 @@ public List getObservationsByVariableDbId(String observationVa // search by ObservationVariableDbIds public List getObservationsByVariableDbIds(List observationVariableDbIds, UUID programId) { + if(observationVariableDbIds.isEmpty()) { + return Collections.emptyList(); + } try { BrAPIObservationSearchRequest request = new BrAPIObservationSearchRequest() @@ -95,6 +98,9 @@ public List getObservationsByVariableDbIds(List observ } public List getObservationsByVariableAndBrAPIProgram(String brapiProgramId, UUID programId, List observationVariableDbIds) { + if(observationVariableDbIds.isEmpty()) { + return Collections.emptyList(); + } try { BrAPIObservationSearchRequest request = new BrAPIObservationSearchRequest() diff --git a/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java b/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java index d371c8103..7ba7ff1f5 100644 --- a/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java +++ b/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java @@ -27,17 +27,20 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.client.v2.model.queryParams.core.LocationQueryParams; import org.brapi.client.v2.modules.core.LocationsApi; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.BrApiGeoJSON; import org.brapi.v2.model.core.BrAPILocation; +import org.brapi.v2.model.core.request.BrAPILocationSearchRequest; import org.brapi.v2.model.core.response.BrAPILocationListResponse; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.dao.db.tables.BiUserTable; import org.breedinginsight.dao.db.tables.daos.PlaceDao; import org.breedinginsight.model.*; -import org.breedinginsight.services.brapi.BrAPIProvider; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.utilities.BrAPIDAOUtil; import org.breedinginsight.utilities.Utilities; +import org.jetbrains.annotations.NotNull; import org.jooq.Configuration; import org.jooq.DSLContext; import org.jooq.Record; @@ -45,61 +48,64 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.UUID; +import java.util.*; +import java.util.stream.Collectors; import static org.breedinginsight.dao.db.Tables.*; @Singleton @Slf4j public class ProgramLocationDAO extends PlaceDao { - private DSLContext dsl; - private BrAPIProvider brAPIProvider; - private Gson gson; + private final DSLContext dsl; + private final Gson gson; - @Property(name = "brapi.server.reference-source") - protected String referenceSource; + protected final String referenceSource; + private final BrAPIDAOUtil brAPIDAOUtil; + + private final ProgramDAO programDAO; + private final BrAPIEndpointProvider brAPIEndpointProvider; @Inject - public ProgramLocationDAO(Configuration config, DSLContext dsl, - BrAPIProvider brAPIProvider) { + public ProgramLocationDAO(Configuration config, DSLContext dsl, @Property(name = "brapi.server.reference-source") String referenceSource, BrAPIDAOUtil brAPIDAOUtil, ProgramDAO programDAO, BrAPIEndpointProvider brAPIEndpointProvider) { super(config); this.dsl = dsl; - this.brAPIProvider = brAPIProvider; this.gson = new GsonBuilder() .registerTypeAdapterFactory(new GeometryAdapterFactory()) .create(); + this.referenceSource = referenceSource; + this.brAPIDAOUtil = brAPIDAOUtil; + this.programDAO = programDAO; + this.brAPIEndpointProvider = brAPIEndpointProvider; } // get all active locations by program id - public List getByProgramId(UUID programId) { + public List getByProgramId(UUID programId) throws ApiException { List records = getProgramLocationsQuery() .where(PLACE.PROGRAM_ID.eq(programId).and(PLACE.ACTIVE.eq(true))) .fetch(); - return parseRecords(records); + return parseRecords(records, true); } // get specified program location regardless of active status // path programId must match programId in location - public Optional getById(UUID programId, UUID locationId) { + public Optional getById(UUID programId, UUID locationId, boolean full) throws ApiException { + List locations = getByIds(programId, List.of(locationId), full); - List records = getProgramLocationsQuery() - .where(PLACE.ID.eq(locationId).and(PLACE.PROGRAM_ID.eq(programId))) - .fetch(); + return Utilities.getSingleOptional(locations); + } - List locations = parseRecords(records); - if (locations.size() > 0){ - return Optional.of(locations.get(0)); - } else { - return Optional.empty(); - } + public List getByIds(UUID programId, Collection locationIds, boolean full) throws ApiException { + List records = getProgramLocationsQuery() + .where(PLACE.ID.in(locationIds)) + .and(PLACE.PROGRAM_ID.eq(programId)) + .fetch(); + + return parseRecords(records, full); } private SelectOnConditionStep getProgramLocationsQuery(){ @@ -117,9 +123,10 @@ private SelectOnConditionStep getProgramLocationsQuery(){ .leftJoin(updatedByUser).on(PLACE.UPDATED_BY.eq(updatedByUser.ID)); } - private List parseRecords(List records) { + private List parseRecords(List records, boolean fetchBrAPIObject) throws ApiException { - List resultLocations = new ArrayList<>(); + Map resultLocations = new HashMap<>(); + List locationIds = new ArrayList<>(); BiUserTable createdByUser = BI_USER.as("createdByUser"); BiUserTable updatedByUser = BI_USER.as("updatedByUser"); @@ -132,17 +139,40 @@ private List parseRecords(List records) { location.setTopography(Topography.parseSQLRecord(record)); location.setCreatedByUser(org.breedinginsight.model.User.parseSQLRecord(record, createdByUser)); location.setUpdatedByUser(User.parseSQLRecord(record, updatedByUser)); - resultLocations.add(location); + resultLocations.put(location.getId(), location); + locationIds.add(location.getId()); + } + + if(fetchBrAPIObject && !resultLocations.isEmpty()) { + List brAPILocations = getBrapiLocations(locationIds, resultLocations.values().stream().findFirst().get().getProgramId()); + + if (brAPILocations.size() != resultLocations.size()) { + throw new IllegalStateException("Did not find BrAPI Location objects for each location"); + } + + brAPILocations.forEach(brapiLocation -> { + BrAPIExternalReference externalReference = Utilities.getExternalReference(brapiLocation.getExternalReferences(), referenceSource) + .orElseThrow(() -> new IllegalStateException("No externalReference for BrAPI Location: " + brapiLocation.getLocationDbId())); + + ProgramLocation location = resultLocations.get(UUID.fromString(externalReference.getReferenceID())); + if(location == null) { + throw new IllegalStateException("Did not find BrAPI Location for location: " + location.getId()); + } + location.setLocationDbId(brapiLocation.getLocationDbId()); + }); } - return resultLocations; + return new ArrayList<>(resultLocations.values()); } - public void createProgramLocationBrAPI(ProgramLocation location) { + public void createProgramLocationBrAPI(ProgramLocation location, Program program) { - BrAPIExternalReference externalReference = new BrAPIExternalReference() + BrAPIExternalReference locationIdRef = new BrAPIExternalReference() .referenceID(location.getId().toString()) .referenceSource(referenceSource); + BrAPIExternalReference programIdRef = new BrAPIExternalReference() + .referenceID(location.getProgramId().toString()) + .referenceSource(String.format("%s/%s", referenceSource, ExternalReferenceSource.PROGRAMS.getName())); BrAPILocation brApiLocation = new BrAPILocation() .abbreviation(location.getAbbreviation()) @@ -155,10 +185,10 @@ public void createProgramLocationBrAPI(ProgramLocation location) { .documentationURL(location.getDocumentationUrl()) .environmentType(location.getEnvironmentType() != null ? location.getEnvironmentType().getName() : null) .exposure(location.getExposure()) - .externalReferences(List.of(externalReference)) + .externalReferences(List.of(locationIdRef, programIdRef)) //.instituteAddress() do not keep this in our model //.instituteName() do not keep this in our model - .locationName(location.getName()) + .locationName(Utilities.appendProgramKey(location.getName(), program.getKey())) //.locationType() do not keep this in our model //.siteStatus() do not keep this in our model .slope(location.getSlope() != null ? location.getSlope().toPlainString() : null) @@ -167,42 +197,28 @@ public void createProgramLocationBrAPI(ProgramLocation location) { // POST locations to each brapi service // TODO: If there is a failure after the first brapi service, roll back all before the failure. try { - List locationsAPIs = brAPIProvider.getAllUniqueLocationsAPI(); - for (LocationsApi locationsAPI: locationsAPIs){ - locationsAPI.locationsPost(List.of(brApiLocation)); + LocationsApi locationsAPI = brAPIEndpointProvider.get(programDAO.getCoreClient(location.getProgramId()), LocationsApi.class); + ApiResponse brapiResponse = locationsAPI.locationsPost(List.of(brApiLocation)); + if(brapiResponse.getBody().getResult().getData().size() == 1) { + location.setLocationDbId(brapiResponse.getBody().getResult().getData().get(0).getLocationDbId()); } } catch (ApiException e) { - log.warn(Utilities.generateApiExceptionLogMessage(e)); + log.error(Utilities.generateApiExceptionLogMessage(e)); throw new InternalServerException("Error making BrAPI call", e); } } - public void updateProgramLocationBrAPI(ProgramLocation location) { + public void updateProgramLocationBrAPI(ProgramLocation location, Program program) { - LocationQueryParams searchRequest = new LocationQueryParams() - .externalReferenceID(location.getId().toString()) - .externalReferenceSource(referenceSource); - - // Location goes in all of the clients - // TODO: If there is a failure after the first brapi service, roll back all before the failure. - List locationsAPIs = brAPIProvider.getAllUniqueLocationsAPI(); - for (LocationsApi locationsAPI: locationsAPIs){ - - // Get existing brapi location - ApiResponse brApiLocations; - try { - brApiLocations = locationsAPI.locationsGet(searchRequest); - } catch (ApiException e) { - log.warn(Utilities.generateApiExceptionLogMessage(e)); - throw new HttpServerException("Could not find location in BrAPI service."); - } + try { + List brApiLocations = getBrapiLocations(List.of(location.getId()), location.getProgramId()); - if (brApiLocations.getBody().getResult().getData().size() != 1){ + if (brApiLocations.size() != 1){ throw new HttpServerException("Could not find unique location in BrAPI service."); } - BrAPILocation brApiLocation = brApiLocations.getBody().getResult().getData().get(0); + BrAPILocation brApiLocation = brApiLocations.get(0); //TODO: Need to add archived/not archived when available in brapi brApiLocation.setAbbreviation(location.getAbbreviation()); @@ -217,19 +233,35 @@ public void updateProgramLocationBrAPI(ProgramLocation location) { brApiLocation.setExposure(location.getExposure()); //brApiLocation.setInstituteAddress(); do not keep this in our model //brApiLocation.setInstituteName(); do not keep this in our model - brApiLocation.setLocationName(location.getName()); + brApiLocation.setLocationName(Utilities.appendProgramKey(location.getName(), program.getKey())); //brApiLocation.setLocationType(); do not keep this in our model //brApiLocation.setSiteStatus(); do not keep this in our model brApiLocation.setSlope(location.getSlope() != null ? location.getSlope().toPlainString() : null); brApiLocation.setTopography(location.getTopography() != null ? location.getTopography().getName() : null); - try { - locationsAPI.locationsLocationDbIdPut(brApiLocation.getLocationDbId(), brApiLocation); - } catch (ApiException e) { - log.warn(Utilities.generateApiExceptionLogMessage(e)); - throw new HttpServerException("Could not find location in BrAPI service."); - } + LocationsApi locationsAPI = brAPIEndpointProvider.get(programDAO.getCoreClient(location.getProgramId()), LocationsApi.class); + locationsAPI.locationsLocationDbIdPut(brApiLocation.getLocationDbId(), brApiLocation); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new HttpServerException("Could not find location in BrAPI service."); + } + } + + private List getBrapiLocations(List locationIds, UUID programId) throws ApiException { + if(locationIds.isEmpty()) { + return Collections.emptyList(); } + + BrAPILocationSearchRequest searchRequest = new BrAPILocationSearchRequest() + .externalReferenceIDs(locationIds.stream().map(UUID::toString).collect(Collectors.toList())) + .externalReferenceSources(List.of(referenceSource)); + + // Location goes in all of the clients + // TODO: If there is a failure, roll back all before the failure. + LocationsApi locationsAPI = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), LocationsApi.class); + + // Get existing brapi location + return brAPIDAOUtil.search(locationsAPI::searchLocationsPost, locationsAPI::searchLocationsSearchResultsDbIdGet, searchRequest); } private BrApiGeoJSON getClientGeoJson(ProgramLocation location) { @@ -245,4 +277,56 @@ private BrApiGeoJSON getClientGeoJson(ProgramLocation location) { } + public List getByDbIds(Collection locationDbIds, UUID programId) throws ApiException { + if(locationDbIds.isEmpty()) { + return Collections.emptyList(); + } + + BrAPILocationSearchRequest searchRequest = new BrAPILocationSearchRequest() + .locationDbIds(new ArrayList<>(locationDbIds)) + .externalReferenceIDs(List.of(programId.toString())) + .externalReferenceSources(List.of(String.format("%s/%s", referenceSource, ExternalReferenceSource.PROGRAMS.getName()))); + + return getProgramLocationsByBrAPISearch(programId, searchRequest); + } + + public List getByNames(List names, UUID programId, boolean full) throws ApiException { + if(names.isEmpty()) { + return Collections.emptyList(); + } + + List records = getProgramLocationsQuery().where(PLACE.NAME.in(names)) + .and(PLACE.PROGRAM_ID.eq(programId)) + .fetch(); + + return parseRecords(records, full); + } + + @NotNull + private List getProgramLocationsByBrAPISearch(UUID programId, BrAPILocationSearchRequest searchRequest) throws ApiException { + LocationsApi locationsAPI = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), LocationsApi.class); + List searchResult = brAPIDAOUtil.search(locationsAPI::searchLocationsPost, locationsAPI::searchLocationsSearchResultsDbIdGet, searchRequest); + + Map brapiLocationById = new HashMap<>(); + searchResult.forEach(brAPILocation -> { + BrAPIExternalReference xref = Utilities.getExternalReference(brAPILocation.getExternalReferences(), referenceSource) + .orElseThrow(() -> new IllegalStateException(String.format("Location (by dbid): %s does not have any external references", brAPILocation.getLocationDbId()))); + brapiLocationById.put(UUID.fromString(xref.getReferenceID()), brAPILocation); + }); + + List records = getProgramLocationsQuery() + .where(PLACE.ID.in(brapiLocationById.keySet()).and(PLACE.PROGRAM_ID.eq(programId))) + .fetch(); + List programLocations = parseRecords(records, false); + if(programLocations.size() != brapiLocationById.size()) { + throw new IllegalStateException("Didn't find all locations by id"); + } + + programLocations.forEach(location -> { + BrAPILocation brAPILocation = brapiLocationById.get(location.getId()); + location.setLocationDbId(brAPILocation.getLocationDbId()); + }); + + return programLocations; + } } diff --git a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java index 4b10d257e..2dba86807 100644 --- a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java +++ b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java @@ -39,7 +39,6 @@ import org.breedinginsight.dao.db.tables.BiUserTable; import org.breedinginsight.dao.db.tables.daos.TraitDao; import org.breedinginsight.dao.db.tables.pojos.TraitEntity; -import org.breedinginsight.dao.db.tables.records.TraitRecord; import org.breedinginsight.daos.ObservationDAO; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.TraitDAO; @@ -62,7 +61,6 @@ import java.util.stream.Stream; import static org.breedinginsight.dao.db.Tables.*; -import static org.breedinginsight.services.brapi.BrAPIClientType.PHENO; import static org.jooq.impl.DSL.lower; @Singleton @@ -313,7 +311,9 @@ public List getObservationsForTraitsByBrAPIProgram(String brap @Override public List searchVariables(List variableIds, UUID programId) { - if (variableIds == null || variableIds.size() == 0) return new ArrayList<>(); + if (variableIds == null || variableIds.size() == 0) { + return Collections.emptyList(); + } try { BrAPIObservationVariableSearchRequest request = new BrAPIObservationVariableSearchRequest() .externalReferenceIDs(variableIds); @@ -442,10 +442,8 @@ public List createTraitsBrAPI(List traits, User actingUser, Progra // TODO: If there is a failure after the first brapi service, roll back all before the failure. ApiResponse createdVariables = null; try { - List variablesAPIS = brAPIProvider.getAllUniqueVariablesAPI(); - for (ObservationVariablesApi variablesAPI: variablesAPIS){ - createdVariables = variablesAPI.variablesPost(brApiVariables); - } + ObservationVariablesApi variablesAPI = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationVariablesApi.class); + createdVariables = variablesAPI.variablesPost(brApiVariables); } catch (ApiException e) { log.warn(Utilities.generateApiExceptionLogMessage(e)); throw new InternalServerException("Error making BrAPI call", e); diff --git a/src/main/java/org/breedinginsight/daos/impl/UserDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/UserDAOImpl.java index 449a2a742..e179bc42d 100644 --- a/src/main/java/org/breedinginsight/daos/impl/UserDAOImpl.java +++ b/src/main/java/org/breedinginsight/daos/impl/UserDAOImpl.java @@ -64,11 +64,7 @@ public Optional getUser(UUID id) { List programUsers = programUserDAO.getProgramUsersByUserId(id); List users = parseRecords(records, programUsers); - if (users.size() > 0){ - return Optional.of(users.get(0)); - } else { - return Optional.empty(); - } + return Utilities.getSingleOptional(users); } public Optional getUserByOrcId(String orcid) { @@ -78,11 +74,7 @@ public Optional getUserByOrcId(String orcid) { List programUsers = programUserDAO.getProgramUsersByOrcid(orcid); List users = parseRecords(records, programUsers); - if (users.size() > 0){ - return Optional.of(users.get(0)); - } else { - return Optional.empty(); - } + return Utilities.getSingleOptional(users); } private SelectOnConditionStep getUsersQuery(){ diff --git a/src/main/java/org/breedinginsight/db/migration/V1_0_13__Update_BrAPI_Locations_XRefs.java b/src/main/java/org/breedinginsight/db/migration/V1_0_13__Update_BrAPI_Locations_XRefs.java new file mode 100644 index 000000000..e4285f248 --- /dev/null +++ b/src/main/java/org/breedinginsight/db/migration/V1_0_13__Update_BrAPI_Locations_XRefs.java @@ -0,0 +1,124 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.db.migration; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.ApiResponse; +import org.brapi.client.v2.BrAPIClient; +import org.brapi.client.v2.model.queryParams.core.LocationQueryParams; +import org.brapi.client.v2.modules.core.LocationsApi; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPILocation; +import org.brapi.v2.model.core.response.BrAPILocationListResponse; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.utilities.Utilities; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.*; + +@Slf4j +public class V1_0_13__Update_BrAPI_Locations_XRefs extends BaseJavaMigration { + + final private String DEFAULT_URL_KEY = "default-url"; + final private String BRAPI_REFERENCE_SOURCE_KEY = "brapi-reference-source"; + + public void migrate(Context context) throws Exception { + Map placeholders = context.getConfiguration().getPlaceholders(); + String defaultUrl = placeholders.get(DEFAULT_URL_KEY); + String referenceSource = placeholders.get(BRAPI_REFERENCE_SOURCE_KEY); + + // Get all the programs + List programs = getAllPrograms(context, defaultUrl); + Map locationsApiForProgram = new HashMap<>(); + for (Program program : programs) { + BrAPIClient client = new BrAPIClient(program.getBrapiUrl(), 240000); + locationsApiForProgram.put(program.getId(), new LocationsApi(client)); + } + + List locations = getAllLocations(context); + // Process all the locations in the system + String locationReferenceSource = String.format("%s", referenceSource); + for(ProgramLocation location : locations) { + log.debug("Migrating BrAPI locations missing program external reference for locationId: " + location.getId()); + + String programReferenceSource = String.format("%s/%s", referenceSource, ExternalReferenceSource.PROGRAMS.getName()); + var api = locationsApiForProgram.get(location.getProgramId()); + + LocationQueryParams queryParams = new LocationQueryParams(); + queryParams.externalReferenceSource(locationReferenceSource); + queryParams.externalReferenceID(location.getId().toString()); + queryParams.page(0); + queryParams.pageSize(1000); + ApiResponse locationsResponse = api.locationsGet(queryParams); + + if(locationsResponse.getBody().getResult().getData().size() == 1) { + BrAPILocation brapiLocation = locationsResponse.getBody().getResult().getData().get(0); + Optional programRef = Utilities.getExternalReference(brapiLocation.getExternalReferences(), programReferenceSource); + + if(programRef.isEmpty()) { + BrAPIExternalReference programIdRef = new BrAPIExternalReference() + .referenceID(location.getProgramId().toString()) + .referenceSource(String.format("%s/%s", referenceSource, ExternalReferenceSource.PROGRAMS.getName())); + brapiLocation.getExternalReferences().add(programIdRef); + + api.locationsLocationDbIdPut(brapiLocation.getLocationDbId(), brapiLocation); + } + } + } + log.debug("Done updating locations"); + } + + private List getAllLocations(Context context) throws SQLException { + List locations = new ArrayList<>(); + try (Statement select = context.getConnection().createStatement()) { + try (ResultSet rows = select.executeQuery("SELECT * FROM place")) { + while (rows.next()) { + ProgramLocation location = new ProgramLocation(); + location.setId(UUID.fromString(rows.getString("id"))); + location.setProgramId(UUID.fromString(rows.getString("program_id"))); + locations.add(location); + } + } + } + return locations; + } + + private List getAllPrograms(Context context, String defaultUrl) throws Exception { + List programs = new ArrayList<>(); + try (Statement select = context.getConnection().createStatement()) { + try (ResultSet rows = select.executeQuery("SELECT id, brapi_url, key FROM program where active = true ORDER BY id")) { + while (rows.next()) { + Program program = new Program(); + program.setId(UUID.fromString(rows.getString(1))); + String brapi_url = rows.getString(2); + if (brapi_url == null) brapi_url = defaultUrl; + program.setBrapiUrl(brapi_url); + program.setKey(rows.getString(3)); + programs.add(program); + } + } + } + return programs; + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/model/BrAPIConstants.java b/src/main/java/org/breedinginsight/model/BrAPIConstants.java index ab767ff39..dde083154 100644 --- a/src/main/java/org/breedinginsight/model/BrAPIConstants.java +++ b/src/main/java/org/breedinginsight/model/BrAPIConstants.java @@ -4,7 +4,7 @@ public enum BrAPIConstants { SYSTEM_DEFAULT("System Default"), - REPLICATE( "replicate"), + REPLICATE( "rep"), BLOCK( "block"); private String value; diff --git a/src/main/java/org/breedinginsight/model/ProgramLocation.java b/src/main/java/org/breedinginsight/model/ProgramLocation.java index 05ff52d4d..0067b5d11 100644 --- a/src/main/java/org/breedinginsight/model/ProgramLocation.java +++ b/src/main/java/org/breedinginsight/model/ProgramLocation.java @@ -45,6 +45,7 @@ public class ProgramLocation extends PlaceEntity { private Accessibility accessibility; private EnvironmentType environmentType; private Topography topography; + private String locationDbId; private User createdByUser; private User updatedByUser; diff --git a/src/main/java/org/breedinginsight/services/ProgramLocationService.java b/src/main/java/org/breedinginsight/services/ProgramLocationService.java index da78aa6e2..8ebd85959 100644 --- a/src/main/java/org/breedinginsight/services/ProgramLocationService.java +++ b/src/main/java/org/breedinginsight/services/ProgramLocationService.java @@ -20,10 +20,12 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; import org.breedinginsight.dao.db.tables.pojos.PlaceEntity; import org.breedinginsight.daos.ProgramLocationDAO; +import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; @@ -35,9 +37,7 @@ import javax.inject.Inject; import javax.inject.Singleton; import java.time.OffsetDateTime; -import java.util.List; -import java.util.Optional; -import java.util.UUID; +import java.util.*; @Slf4j @Singleton @@ -68,7 +68,7 @@ public ProgramLocationService(ProgramLocationDAO programLocationDao, this.dsl = dsl; } - public List getByProgramId(UUID programId) throws DoesNotExistException { + public List getByProgramId(UUID programId) throws DoesNotExistException, ApiException { if (!programService.exists(programId)) { throw new DoesNotExistException("Program id does not exist"); @@ -77,9 +77,8 @@ public List getByProgramId(UUID programId) throws DoesNotExistE return programLocationDao.getByProgramId(programId); } - public Optional getById(UUID programId, UUID locationId) { - - return programLocationDao.getById(programId, locationId); + public Optional getById(UUID programId, UUID locationId) throws ApiException { + return programLocationDao.getById(programId, locationId, false); } private UUID validateCountryId(ProgramLocationRequest programLocationRequest) throws UnprocessableEntityException { @@ -195,56 +194,69 @@ boolean isListPointsLatLngValid(List points) { return true; } + public List create(AuthenticatedUser actingUser, UUID programId, List newLocations) throws MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + // check if programId exists + Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); + + List ret = new ArrayList<>(); + for(ProgramLocationRequest newLoc : newLocations) { + ret.add(createLocation(actingUser, program, newLoc)); + } + + return ret; + } + public ProgramLocation create(AuthenticatedUser actingUser, UUID programId, ProgramLocationRequest programLocationRequest) throws DoesNotExistException, MissingRequiredInfoException, UnprocessableEntityException { // check if programId exists - if (!programService.exists(programId)) { - throw new DoesNotExistException("Program id does not exist"); - } + Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); + + return createLocation(actingUser, program, programLocationRequest); + } + private ProgramLocation createLocation(AuthenticatedUser actingUser, Program program, ProgramLocationRequest programLocationRequest) throws UnprocessableEntityException, MissingRequiredInfoException, DoesNotExistException { // validate fields UUID countryId = validateCountryId(programLocationRequest); UUID environmentTypeId = validateEnvironmentTypeId(programLocationRequest); UUID accessibilityId = validateAccessibilityId(programLocationRequest); UUID topographyId = validateTopographyId(programLocationRequest); String coordinates = validateCoordinates(programLocationRequest); + UUID programId = program.getId(); // parse and create the program location object PlaceEntity placeEntity = PlaceEntity.builder() - .programId(programId) - .name(programLocationRequest.getName()) - .countryId(countryId) - .environmentTypeId(environmentTypeId) - .accessibilityId(accessibilityId) - .topographyId(topographyId) - .abbreviation(programLocationRequest.getAbbreviation()) - .coordinates(coordinates != null ? JSONB.valueOf(coordinates) : null) - .coordinateUncertainty(programLocationRequest.getCoordinateUncertainty()) - .coordinateDescription(programLocationRequest.getCoordinateDescription()) - .slope(programLocationRequest.getSlope()) - .exposure(programLocationRequest.getExposure()) - .documentationUrl(programLocationRequest.getDocumentationUrl()) - .createdBy(actingUser.getId()) - .updatedBy(actingUser.getId()) - .build(); - - - ProgramLocation location = null; + .programId(programId) + .name(programLocationRequest.getName()) + .countryId(countryId) + .environmentTypeId(environmentTypeId) + .accessibilityId(accessibilityId) + .topographyId(topographyId) + .abbreviation(programLocationRequest.getAbbreviation()) + .coordinates(coordinates != null ? JSONB.valueOf(coordinates) : null) + .coordinateUncertainty(programLocationRequest.getCoordinateUncertainty()) + .coordinateDescription(programLocationRequest.getCoordinateDescription()) + .slope(programLocationRequest.getSlope()) + .exposure(programLocationRequest.getExposure()) + .documentationUrl(programLocationRequest.getDocumentationUrl()) + .createdBy(actingUser.getId()) + .updatedBy(actingUser.getId()) + .build(); + + // Insert and update // This is warped in a transaction so if the BrAPI save call fails, the BI database insert is rolled back. - location = dsl.transactionResult(configuration -> { + return dsl.transactionResult(configuration -> { programLocationDao.insert(placeEntity); - ProgramLocation progLocation = programLocationDao.getById(programId, placeEntity.getId()).get(); + ProgramLocation progLocation = programLocationDao.getById(programId, placeEntity.getId(), false).orElseThrow(() -> new IllegalStateException("Location appears to not have been created")); // Add location to brapi service - programLocationDao.createProgramLocationBrAPI(progLocation); + programLocationDao.createProgramLocationBrAPI(progLocation, program); return progLocation; }); - return location; } public ProgramLocation update(AuthenticatedUser actingUser, @@ -253,9 +265,7 @@ public ProgramLocation update(AuthenticatedUser actingUser, ProgramLocationRequest programLocationRequest) throws DoesNotExistException, MissingRequiredInfoException, UnprocessableEntityException { - if (!programService.exists(programId)) { - throw new DoesNotExistException("Program id does not exist"); - } + Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); PlaceEntity placeEntity = programLocationDao.fetchOneById(locationId); if (placeEntity == null || (!placeEntity.getProgramId().equals(programId))){ @@ -285,10 +295,10 @@ public ProgramLocation update(AuthenticatedUser actingUser, // This is warped in a transaction so if the BrAPI update post fails, the BI database update is rolled back. ProgramLocation location = dsl.transactionResult(configuration -> { programLocationDao.update(placeEntity); - ProgramLocation progLocation = programLocationDao.getById(programId, placeEntity.getId()).get(); + ProgramLocation progLocation = programLocationDao.getById(programId, placeEntity.getId(), false).get(); // Update location in brapi service - programLocationDao.updateProgramLocationBrAPI(progLocation); + programLocationDao.updateProgramLocationBrAPI(progLocation, program); return progLocation; }); return location; @@ -321,4 +331,12 @@ public void delete(UUID locationId) throws DoesNotExistException { programLocationDao.delete(placeEntity); } + + public List getLocationsByDbId(Collection locationDbIds, UUID programId) throws ApiException { + return programLocationDao.getByDbIds(locationDbIds, programId); + } + + public List getLocationsByName(List names, UUID programId) throws ApiException { + return programLocationDao.getByNames(names, programId, true); + } } diff --git a/src/main/java/org/breedinginsight/services/ProgramService.java b/src/main/java/org/breedinginsight/services/ProgramService.java index 6a4e52238..d7b35801b 100644 --- a/src/main/java/org/breedinginsight/services/ProgramService.java +++ b/src/main/java/org/breedinginsight/services/ProgramService.java @@ -95,6 +95,21 @@ public Optional getById(UUID programId) { return Optional.of(program); } + public Optional getByKey(String programKey) { + + List programs = dao.getProgramByKey(programKey); + + if (programs.size() <= 0) { + return Optional.empty(); + } + + Program program = programs.get(0); + BrAPIProgram brapiProgram = dao.getProgramBrAPI(program); + program.setBrAPIProperties(brapiProgram); + + return Optional.of(program); + } + public List getAll(AuthenticatedUser actingUser){ /* Get all of the programs the user has access to */ List enrolledProgramIds = securityService.getEnrolledProgramIds(actingUser); diff --git a/src/main/java/org/breedinginsight/services/exceptions/ValidatorException.java b/src/main/java/org/breedinginsight/services/exceptions/ValidatorException.java index d0a194bc2..f07c1a3df 100644 --- a/src/main/java/org/breedinginsight/services/exceptions/ValidatorException.java +++ b/src/main/java/org/breedinginsight/services/exceptions/ValidatorException.java @@ -18,9 +18,11 @@ package org.breedinginsight.services.exceptions; import lombok.Getter; +import lombok.ToString; import org.breedinginsight.api.model.v1.response.ValidationErrors; @Getter +@ToString public class ValidatorException extends Exception { private ValidationErrors errors; diff --git a/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java b/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java index fc975d702..73fb4c128 100644 --- a/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java +++ b/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java @@ -387,7 +387,7 @@ private List fetchSamples(BrAPIClient genoBrAPIClient, Program prog log.debug("fetching samples for OUs"); if(observationUnits.isEmpty()) { log.debug("No OUs were supplied, returning"); - return new ArrayList<>(); + return Collections.emptyList(); } SamplesApi samplesApi = brAPIEndpointProvider.get(genoBrAPIClient, SamplesApi.class); @@ -451,7 +451,7 @@ private List fetchCallsets(BrAPIClient genoBrAPIClient, List(); + return Collections.emptyList(); } CallSetsApi callSetsApi = brAPIEndpointProvider.get(genoBrAPIClient, CallSetsApi.class); @@ -466,7 +466,7 @@ private List fetchCalls(BrAPIClient genoBrAPIClient, List(); + return Collections.emptyList(); } CallsApi callsApi = brAPIEndpointProvider.get(genoBrAPIClient, CallsApi.class); @@ -480,7 +480,7 @@ private List fetchVariants(BrAPIClient genoBrAPIClient, List(); + return Collections.emptyList(); } List variantIds = calls.stream() .map(BrAPICall::getVariantDbId) diff --git a/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java b/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java index 018431fc6..2756f43ce 100644 --- a/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java +++ b/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java @@ -49,7 +49,11 @@ public List fetchBreedingMethodsInUse(UUID programI //TODO retest with new germplasm after updating the DAO to return the correct ID for a method germplasmService.getGermplasm(programId).forEach(germplasm -> { UUID id = UUID.fromString(germplasm.getBreedingMethodDbId()); - inUse.put(id, programMethods.get(id)); + if(programMethods.containsKey(id)) { + inUse.put(id, programMethods.get(id)); + } else { + throw new IllegalStateException("Could not find breeding method by id: " + id); + } }); return new ArrayList<>(inUse.values()); diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index b8001ac2e..a74235c4d 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -31,6 +31,7 @@ import org.brapi.v2.model.*; import org.breedinginsight.brapps.importer.model.ImportUpload; +import javax.inject.Inject; import javax.inject.Singleton; import java.time.Duration; import java.util.ArrayList; @@ -43,14 +44,21 @@ @Slf4j public class BrAPIDAOUtil { - @Property(name = "brapi.search.wait-time") - private int searchWaitTime; - @Property(name = "brapi.read-timeout") - private Duration searchTimeout; - @Property(name = "brapi.page-size") - private int pageSize; - @Property(name = "brapi.post-group-size") - private int postGroupSize; + private final int searchWaitTime; + private final Duration searchTimeout; + private final int pageSize; + private final int postGroupSize; + + @Inject + public BrAPIDAOUtil(@Property(name = "brapi.search.wait-time") int searchWaitTime, + @Property(name = "brapi.read-timeout") Duration searchTimeout, + @Property(name = "brapi.page-size") int pageSize, + @Property(name = "brapi.post-group-size") int postGroupSize) { + this.searchWaitTime = searchWaitTime; + this.searchTimeout = searchTimeout; + this.pageSize = pageSize; + this.postGroupSize = postGroupSize; + } public List search(Function, Optional>>> searchMethod, Function3, Optional>>> searchGetMethod, @@ -280,7 +288,9 @@ public List post(List brapiObjects, } List data = result.getData(); // TODO: Maybe move this outside of the loop - if (data.size() != postChunk.size()) throw new ApiException("Number of brapi objects returned does not equal number sent"); + if (data.size() != postChunk.size()) { + throw new ApiException("Number of brapi objects returned does not equal number sent"); + } listResult.addAll(data); finished += data.size(); currentRightBorder += postGroupSize; diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 12084c9de..5384a8241 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -19,6 +19,7 @@ import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.germ.BrAPIGermplasmSynonyms; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; @@ -59,7 +60,7 @@ public static boolean containsCaseInsensitive(String target, List list){ * @return the formatted string */ public static String appendProgramKey(String original, String programKey, String additionalKeyData) { - if(StringUtils.isNotEmpty(additionalKeyData)) { + if(StringUtils.isNotBlank(additionalKeyData)) { return String.format("%s [%s-%s]", original, programKey, additionalKeyData); } else { return String.format("%s [%s]", original, programKey); @@ -79,7 +80,7 @@ public static String appendProgramKey( String original, String programKey ){ * @return */ public static String removeProgramKey(String original, String programKey, String additionalKeyData) { - if(StringUtils.isNotEmpty(additionalKeyData)) { + if(StringUtils.isNotBlank(additionalKeyData)) { String keyValue = String.format(" [%s-%s]", programKey, additionalKeyData); return original.replace(keyValue, ""); } else { @@ -120,4 +121,24 @@ public static String generateApiExceptionLogMessage(ApiException e) { public static String generateReferenceSource(String referenceSourceBase, ExternalReferenceSource referenceSource) { return String.format("%s/%s",referenceSourceBase, referenceSource.getName()); } + + public static Optional getExternalReference(List externalReferences, String source) { + if(externalReferences == null) { + return Optional.empty(); + } + return externalReferences.stream().filter(externalReference -> externalReference.getReferenceSource().equals(source)).findFirst(); + } + + /** + * For a list of items, if the list has only one item, return that item, otherwise return an empty {@link Optional} + * @param items {@link List} of items + * @return Optional of type T or empty Optional + */ + public static Optional getSingleOptional(List items) { + if(items.size() == 1) { + return Optional.of(items.get(0)); + } else { + return Optional.empty(); + } + } } diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java new file mode 100644 index 000000000..2e3295a0a --- /dev/null +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -0,0 +1,1019 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer; + +import com.google.gson.*; +import com.google.gson.reflect.TypeToken; +import io.kowalski.fannypack.FannyPack; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import io.reactivex.Flowable; +import lombok.SneakyThrows; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPISeason; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship; +import org.brapi.v2.model.pheno.BrAPIPositionCoordinateTypeEnum; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.TestUtils; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramRequest; +import org.breedinginsight.api.model.v1.request.SpeciesRequest; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.daos.*; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.dao.db.enums.DataType; +import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; +import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.SpeciesDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.*; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.ProgramLocationService; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.services.SpeciesService; +import org.breedinginsight.services.exceptions.BadRequestException; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.services.writers.CSVWriter; +import org.breedinginsight.utilities.Utilities; +import org.jooq.DSLContext; +import org.junit.jupiter.api.*; +import org.junit.platform.commons.util.StringUtils; + +import javax.inject.Inject; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.*; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.junit.jupiter.api.Assertions.*; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class ExperimentFileImportTest extends BrAPITest { + + private FannyPack securityFp; + private String mappingId; + private BiUserEntity testUser; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + @Property(name = "brapi.server.core-url") + private String BRAPI_URL; + + @Inject + private SpeciesService speciesService; + @Inject + private UserDAO userDAO; + @Inject + private DSLContext dsl; + + @Inject + private SpeciesDAO speciesDAO; + + @Inject + private ProgramDAO programDAO; + + @Inject + private ProgramService programService; + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + private ImportTestUtils importTestUtils; + + @Inject + private OntologyService ontologyService; + + @Inject + private BrAPITrialDAO brAPITrialDAO; + + @Inject + private BrAPIStudyDAO brAPIStudyDAO; + + @Inject + private BrAPIObservationUnitDAO ouDAO; + + @Inject + private ProgramLocationService locationService; + + @Inject + private BrAPIGermplasmDAO germplasmDAO; + + @Inject + private BrAPIObservationDAO observationDAO; + + @Inject + private BrAPISeasonDAO seasonDAO; + + private Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .create(); + + @BeforeAll + public void setup() { + importTestUtils = new ImportTestUtils(); + Map setupObjects = importTestUtils.setup(client, gson, dsl, speciesService, userDAO, super.getBrapiDsl(), "ExperimentsTemplateMap"); + mappingId = (String) setupObjects.get("mappingId"); + testUser = (BiUserEntity) setupObjects.get("testUser"); + securityFp = (FannyPack) setupObjects.get("securityFp"); + + } + + /* + Tests + - new experiment + - existing experiment, new env + - new env, existing location + - new experiment, missing required cols (check all required cols) + - new env, missing required cols (check all req cols) + - new exp/env with observations + - new exp/env with invalid observations + - existing env with missing OU ID + - existing env with new observations + - existing env that already has obs + */ + + @Test + @SneakyThrows + public void importNewExpNewLocNoObsSuccess() { + Program program = createProgram("New Exp and Loc", "NEXPL", "NEXPL", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + Map validRow = new HashMap<>(); + validRow.put(Columns.GERMPLASM_GID, "1"); + validRow.put(Columns.TEST_CHECK, "T"); + validRow.put(Columns.EXP_TITLE, "Test Exp"); + validRow.put(Columns.EXP_DESCRIPTION, "Test Description"); + validRow.put(Columns.EXP_UNIT, "Plot"); + validRow.put(Columns.EXP_TYPE, "Phenotyping"); + validRow.put(Columns.ENV, "Test Env"); + validRow.put(Columns.ENV_LOCATION, "Location A"); + validRow.put(Columns.ENV_YEAR, "2023"); + validRow.put(Columns.EXP_UNIT_ID, "a-1"); + validRow.put(Columns.REP_NUM, "1"); + validRow.put(Columns.BLOCK_NUM, "1"); + validRow.put(Columns.ROW, "1"); + validRow.put(Columns.COLUMN, "1"); + validRow.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); + + Flowable> call = importTestUtils.uploadDataFile(writeDataToFile(List.of(validRow), null), null, true, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("NEW", row.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("study").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(validRow, program, null); + } + + @Test + @SneakyThrows + public void importNewExpMultiNewEnvNoObsSuccess() { + Program program = createProgram("New Exp and Multi New Env", "MULENV", "MULENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + Map firstEnv = new HashMap<>(); + firstEnv.put(Columns.GERMPLASM_GID, "1"); + firstEnv.put(Columns.TEST_CHECK, "T"); + firstEnv.put(Columns.EXP_TITLE, "Test Exp"); + firstEnv.put(Columns.EXP_DESCRIPTION, "Test Description"); + firstEnv.put(Columns.EXP_UNIT, "Plot"); + firstEnv.put(Columns.EXP_TYPE, "Phenotyping"); + firstEnv.put(Columns.ENV, "Test Env A"); + firstEnv.put(Columns.ENV_LOCATION, "Location A"); + firstEnv.put(Columns.ENV_YEAR, "2023"); + firstEnv.put(Columns.EXP_UNIT_ID, "a-1"); + firstEnv.put(Columns.REP_NUM, "1"); + firstEnv.put(Columns.BLOCK_NUM, "1"); + firstEnv.put(Columns.ROW, "1"); + firstEnv.put(Columns.COLUMN, "1"); + firstEnv.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); + + Map secondEnv = new HashMap<>(); + secondEnv.put(Columns.GERMPLASM_GID, "1"); + secondEnv.put(Columns.TEST_CHECK, "T"); + secondEnv.put(Columns.EXP_TITLE, "Test Exp"); + secondEnv.put(Columns.EXP_DESCRIPTION, "Test Description"); + secondEnv.put(Columns.EXP_UNIT, "Plot"); + secondEnv.put(Columns.EXP_TYPE, "Phenotyping"); + secondEnv.put(Columns.ENV, "Test Env B"); + secondEnv.put(Columns.ENV_LOCATION, "Location B"); + secondEnv.put(Columns.ENV_YEAR, "2023"); + secondEnv.put(Columns.EXP_UNIT_ID, "b-1"); + secondEnv.put(Columns.REP_NUM, "1"); + secondEnv.put(Columns.BLOCK_NUM, "1"); + secondEnv.put(Columns.ROW, "1"); + secondEnv.put(Columns.COLUMN, "1"); + secondEnv.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); + + Flowable> call = importTestUtils.uploadDataFile(writeDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(2, previewRows.size()); + JsonObject firstRow = previewRows.get(0).getAsJsonObject(); + + assertEquals("NEW", firstRow.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("NEW", firstRow.getAsJsonObject("location").get("state").getAsString()); + assertEquals("NEW", firstRow.getAsJsonObject("study").get("state").getAsString()); + assertEquals("NEW", firstRow.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(firstEnv, program, null); + + JsonObject secondRow = previewRows.get(0).getAsJsonObject(); + + assertEquals("NEW", secondRow.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("NEW", secondRow.getAsJsonObject("location").get("state").getAsString()); + assertEquals("NEW", secondRow.getAsJsonObject("study").get("state").getAsString()); + assertEquals("NEW", secondRow.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(secondEnv, program, null); + } + + @Test + @SneakyThrows + public void importNewEnvExistingExpNoObsSuccess() { + Program program = createProgram("New Env Existing Exp", "NEWENV", "NEWENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + + importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + + Map newEnv = new HashMap<>(); + newEnv.put(Columns.GERMPLASM_GID, "1"); + newEnv.put(Columns.TEST_CHECK, "T"); + newEnv.put(Columns.EXP_TITLE, "Test Exp"); + newEnv.put(Columns.EXP_UNIT, "Plot"); + newEnv.put(Columns.EXP_TYPE, "Phenotyping"); + newEnv.put(Columns.ENV, "New Trial Existing Exp"); + newEnv.put(Columns.ENV_LOCATION, "Location A"); + newEnv.put(Columns.ENV_YEAR, "2023"); + newEnv.put(Columns.EXP_UNIT_ID, "a-1"); + newEnv.put(Columns.REP_NUM, "1"); + newEnv.put(Columns.BLOCK_NUM, "1"); + newEnv.put(Columns.ROW, "1"); + newEnv.put(Columns.COLUMN, "1"); + + JsonObject result = importTestUtils.uploadAndFetch(writeDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("study").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(newEnv, program, null); + } + + @Test + @SneakyThrows + public void verifyMissingDataThrowsError() { + Program program = createProgram("Missing Req Cols", "MISS", "MISS", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + + Map base = new HashMap<>(); + base.put(Columns.GERMPLASM_GID, "1"); + base.put(Columns.TEST_CHECK, "T"); + base.put(Columns.EXP_TITLE, "Missing Req Cols"); + base.put(Columns.EXP_UNIT, "Plot"); + base.put(Columns.EXP_TYPE, "Phenotyping"); + base.put(Columns.ENV, "Missing GID"); + base.put(Columns.ENV_LOCATION, "Location A"); + base.put(Columns.ENV_YEAR, "2023"); + base.put(Columns.EXP_UNIT_ID, "a-1"); + base.put(Columns.REP_NUM, "1"); + base.put(Columns.BLOCK_NUM, "1"); + + Map noGID = new HashMap<>(base); + noGID.remove(Columns.GERMPLASM_GID); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID); + + Map noExpTitle = new HashMap<>(base); + noExpTitle.remove(Columns.EXP_TITLE); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE); + + Map noExpUnit = new HashMap<>(base); + noExpUnit.remove(Columns.EXP_UNIT); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT); + + Map noExpType = new HashMap<>(base); + noExpType.remove(Columns.EXP_TYPE); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noExpType), null), Columns.EXP_TYPE); + + Map noEnv = new HashMap<>(base); + noEnv.remove(Columns.ENV); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noEnv), null), Columns.ENV); + + Map noEnvLoc = new HashMap<>(base); + noEnvLoc.remove(Columns.ENV_LOCATION); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION); + + Map noExpUnitId = new HashMap<>(base); + noExpUnitId.remove(Columns.EXP_UNIT_ID); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID); + + Map noExpRep = new HashMap<>(base); + noExpRep.remove(Columns.REP_NUM); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noExpRep), null), Columns.REP_NUM); + + Map noExpBlock = new HashMap<>(base); + noExpBlock.remove(Columns.BLOCK_NUM); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM); + + Map noEnvYear = new HashMap<>(base); + noEnvYear.remove(Columns.ENV_YEAR); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR); + } + + @Test + @SneakyThrows + public void importNewExpWithObs() { + List traits = createTraits(1); + Program program = createProgram("New Exp with Observations", "EXPOBS", "EXPOBS", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), "1"); + + JsonObject result = importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("NEW", row.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("study").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(newExp, program, traits); + } + + @Test + @SneakyThrows + public void verifyFailureImportNewExpWithInvalidObs() { + List traits = createTraits(1); + Program program = createProgram("Invalid Observations", "INVOBS", "INVOBS", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), "Red"); + + uploadAndVerifyFailure(program, writeDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName()); + } + + @Test + @SneakyThrows + public void verifyFailureNewOuExistingEnv() { + Program program = createProgram("New OU Exising Env", "FAILOU", "FAILOU", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + + importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + + Map newOU = new HashMap<>(newExp); + newOU.put(Columns.EXP_UNIT_ID, "a-2"); + newOU.put(Columns.ROW, "1"); + newOU.put(Columns.COLUMN, "2"); + + Flowable> call = importTestUtils.uploadDataFile(writeDataToFile(List.of(newOU), null), null, true, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Units are missing Observation Unit Id.")); + } + + @Test + @SneakyThrows + public void importNewObsExisingOu() { + List traits = createTraits(1); + Program program = createProgram("New Obs Existing OU", "OUOBS", "OUOBS", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + + importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of(Utilities.appendProgramKey((String)newExp.get(Columns.EXP_TITLE), program.getKey())), program).get(0); + Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); + assertTrue(trialIdXref.isPresent()); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); + + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); + Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); + assertTrue(ouIdXref.isPresent()); + + Map newObservation = new HashMap<>(); + newObservation.put(Columns.GERMPLASM_GID, "1"); + newObservation.put(Columns.TEST_CHECK, "T"); + newObservation.put(Columns.EXP_TITLE, "Test Exp"); + newObservation.put(Columns.EXP_UNIT, "Plot"); + newObservation.put(Columns.EXP_TYPE, "Phenotyping"); + newObservation.put(Columns.ENV, "New Env"); + newObservation.put(Columns.ENV_LOCATION, "Location A"); + newObservation.put(Columns.ENV_YEAR, "2023"); + newObservation.put(Columns.EXP_UNIT_ID, "a-1"); + newObservation.put(Columns.REP_NUM, "1"); + newObservation.put(Columns.BLOCK_NUM, "1"); + newObservation.put(Columns.ROW, "1"); + newObservation.put(Columns.COLUMN, "1"); + newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); + newObservation.put(traits.get(0).getObservationVariableName(), "1"); + + JsonObject result = importTestUtils.uploadAndFetch(writeDataToFile(List.of(newObservation), traits), null, true, client, program, mappingId); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(newObservation, program, traits); + } + + @Test + @SneakyThrows + public void verifyFailureImportNewObsExisingOuWithExistingObs() { + List traits = createTraits(1); + Program program = createProgram("New Obs Existing Obs", "EXOBS", "EXOBS", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), "1"); + + importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of(Utilities.appendProgramKey((String)newExp.get(Columns.EXP_TITLE), program.getKey())), program).get(0); + Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); + assertTrue(trialIdXref.isPresent()); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); + + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); + Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); + assertTrue(ouIdXref.isPresent()); + + Map newObservation = new HashMap<>(); + newObservation.put(Columns.GERMPLASM_GID, "1"); + newObservation.put(Columns.TEST_CHECK, "T"); + newObservation.put(Columns.EXP_TITLE, "Test Exp"); + newObservation.put(Columns.EXP_UNIT, "Plot"); + newObservation.put(Columns.EXP_TYPE, "Phenotyping"); + newObservation.put(Columns.ENV, "New Env"); + newObservation.put(Columns.ENV_LOCATION, "Location A"); + newObservation.put(Columns.ENV_YEAR, "2023"); + newObservation.put(Columns.EXP_UNIT_ID, "a-1"); + newObservation.put(Columns.REP_NUM, "1"); + newObservation.put(Columns.BLOCK_NUM, "1"); + newObservation.put(Columns.ROW, "1"); + newObservation.put(Columns.COLUMN, "1"); + newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); + newObservation.put(traits.get(0).getObservationVariableName(), "2"); + + uploadAndVerifyFailure(program, writeDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName()); + } + + /* + Scenario: + - an experiment was created with observations + - a new experiment is created after the first experiment + - verify the second experiment gets created successfully + */ + @Test + @SneakyThrows + public void importSecondExpAfterFirstExpWithObs() { + List traits = createTraits(1); + Program program = createProgram("New Exp After First", "NEAF", "NEAF", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExpA = new HashMap<>(); + newExpA.put(Columns.GERMPLASM_GID, "1"); + newExpA.put(Columns.TEST_CHECK, "T"); + newExpA.put(Columns.EXP_TITLE, "Test Exp A"); + newExpA.put(Columns.EXP_UNIT, "Plot"); + newExpA.put(Columns.EXP_TYPE, "Phenotyping"); + newExpA.put(Columns.ENV, "New Env"); + newExpA.put(Columns.ENV_LOCATION, "Location A"); + newExpA.put(Columns.ENV_YEAR, "2023"); + newExpA.put(Columns.EXP_UNIT_ID, "a-1"); + newExpA.put(Columns.REP_NUM, "1"); + newExpA.put(Columns.BLOCK_NUM, "1"); + newExpA.put(Columns.ROW, "1"); + newExpA.put(Columns.COLUMN, "1"); + newExpA.put(traits.get(0).getObservationVariableName(), "1"); + + JsonObject resultA = importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + + JsonArray previewRowsA = resultA.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRowsA.size()); + JsonObject rowA = previewRowsA.get(0).getAsJsonObject(); + + assertEquals("NEW", rowA.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("NEW", rowA.getAsJsonObject("location").get("state").getAsString()); + assertEquals("NEW", rowA.getAsJsonObject("study").get("state").getAsString()); + assertEquals("NEW", rowA.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(newExpA, program, traits); + + Map newExpB = new HashMap<>(); + newExpB.put(Columns.GERMPLASM_GID, "1"); + newExpB.put(Columns.TEST_CHECK, "T"); + newExpB.put(Columns.EXP_TITLE, "Test Exp B"); + newExpB.put(Columns.EXP_UNIT, "Plot"); + newExpB.put(Columns.EXP_TYPE, "Phenotyping"); + newExpB.put(Columns.ENV, "New Env"); + newExpB.put(Columns.ENV_LOCATION, "Location A"); + newExpB.put(Columns.ENV_YEAR, "2023"); + newExpB.put(Columns.EXP_UNIT_ID, "a-1"); + newExpB.put(Columns.REP_NUM, "1"); + newExpB.put(Columns.BLOCK_NUM, "1"); + newExpB.put(Columns.ROW, "1"); + newExpB.put(Columns.COLUMN, "1"); + newExpB.put(traits.get(0).getObservationVariableName(), "1"); + + JsonObject resultB = importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + + JsonArray previewRowsB = resultB.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRowsB.size()); + JsonObject rowB = previewRowsB.get(0).getAsJsonObject(); + + assertEquals("NEW", rowB.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("EXISTING", rowB.getAsJsonObject("location").get("state").getAsString()); + assertEquals("NEW", rowB.getAsJsonObject("study").get("state").getAsString()); + assertEquals("NEW", rowB.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(newExpB, program, traits); + } + + private Map assertRowSaved(Map expected, Program program, List traits) throws ApiException { + Map ret = new HashMap<>(); + + List trials = brAPITrialDAO.getTrialsByName(List.of(Utilities.appendProgramKey((String)expected.get(Columns.EXP_TITLE), program.getKey())), program); + assertFalse(trials.isEmpty()); + BrAPITrial trial = trials.get(0); + Optional trialIdXref = Utilities.getExternalReference(trial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); + assertTrue(trialIdXref.isPresent()); + + List studies = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program); + assertFalse(studies.isEmpty()); + BrAPIStudy study = null; + for(BrAPIStudy s : studies) { + if(expected.get(Columns.ENV).equals(Utilities.removeProgramKeyAndUnknownAdditionalData(s.getStudyName(), program.getKey()))) { + study = s; + break; + } + } + assertNotNull(study, "Could not find study by name: " + expected.get(Columns.ENV)); + + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(study.getStudyDbId(), program).get(0); + Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); + assertTrue(ouIdXref.isPresent()); + + List locations = locationService.getLocationsByDbId(List.of(study.getLocationDbId()), program.getId()); + assertFalse(locations.isEmpty()); + ProgramLocation location = locations.get(0); + + List germplasms = germplasmDAO.getGermplasmsByDBID(List.of(ou.getGermplasmDbId()), program.getId()); + assertFalse(germplasms.isEmpty()); + BrAPIGermplasm germplasm = germplasms.get(0); + + BrAPISeason season = seasonDAO.getSeasonById(study.getSeasons().get(0), program.getId()); + + ret.put("trial", trial); + ret.put("study", study); + ret.put("location", location); + ret.put("observationUnit", ou); + ret.put("germplasm", germplasm); + + List observations = null; + if(traits != null) { + observations = observationDAO.getObservationsByStudyName(List.of(study.getStudyName()), program); + assertFalse(observations.isEmpty()); + + ret.put("observations", observations); + } + + assertNotNull(germplasm.getGermplasmName()); + assertEquals(expected.get(Columns.GERMPLASM_GID), germplasm.getAccessionNumber()); + if(expected.containsKey(Columns.TEST_CHECK) && StringUtils.isNotBlank((String)expected.get(Columns.TEST_CHECK))) { + assertEquals(expected.get(Columns.TEST_CHECK), + ou.getObservationUnitPosition() + .getEntryType() + .name() + .substring(0, 1)); + } + assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); + assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); + assertEquals(expected.get(Columns.EXP_DESCRIPTION), trial.getTrialDescription()); + assertEquals(expected.get(Columns.EXP_UNIT), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString()); + assertEquals(expected.get(Columns.EXP_UNIT), ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + assertEquals(expected.get(Columns.EXP_TYPE), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); + assertEquals(expected.get(Columns.EXP_TYPE), study.getStudyType()); + assertEquals(expected.get(Columns.ENV), Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); + assertEquals(expected.get(Columns.ENV_LOCATION), Utilities.removeProgramKey(study.getLocationName(), program.getKey())); + assertEquals(expected.get(Columns.ENV_LOCATION), Utilities.removeProgramKey(location.getName(), program.getKey())); + assertEquals(expected.get(Columns.ENV_YEAR), season.getSeasonName()); + assertEquals(expected.get(Columns.EXP_UNIT_ID), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + + BrAPIObservationUnitLevelRelationship rep = null; + BrAPIObservationUnitLevelRelationship block = null; + for (BrAPIObservationUnitLevelRelationship rel : ou.getObservationUnitPosition().getObservationLevelRelationships()) { + if ("rep".equals(rel.getLevelName()) && rep == null) { + rep = rel; + } else if ("block".equals(rel.getLevelName()) && block == null) { + block = rel; + } + } + assertNotNull(rep); + assertNotNull(block); + assertEquals(expected.get(Columns.REP_NUM), rep.getLevelCode()); + assertEquals(expected.get(Columns.BLOCK_NUM), block.getLevelCode()); + if(expected.containsKey(Columns.ROW)) { + assertEquals(expected.get(Columns.ROW), ou.getObservationUnitPosition().getPositionCoordinateX()); + assertEquals(BrAPIPositionCoordinateTypeEnum.GRID_ROW, ou.getObservationUnitPosition().getPositionCoordinateXType()); + } + if(expected.containsKey(Columns.COLUMN)) { + assertEquals(expected.get(Columns.COLUMN), ou.getObservationUnitPosition().getPositionCoordinateY()); + assertEquals(BrAPIPositionCoordinateTypeEnum.GRID_COL, ou.getObservationUnitPosition().getPositionCoordinateYType()); + } + if(expected.containsKey(Columns.TREATMENT_FACTORS) && StringUtils.isNotBlank((String)expected.get(Columns.TREATMENT_FACTORS))) { + assertEquals(expected.get(Columns.TREATMENT_FACTORS), ou.getTreatments().get(0).getFactor()); + } + + if(traits != null) { + List expectedVariableObservation = new ArrayList<>(); + List actualVariableObservation = new ArrayList<>(); + observations.forEach(observation -> actualVariableObservation.add(String.format("%s:%s", Utilities.removeProgramKey(observation.getObservationVariableName(), program.getKey()), observation.getValue()))); + for(Trait trait : traits) { + expectedVariableObservation.add(String.format("%s:%s", trait.getObservationVariableName(), expected.get(trait.getObservationVariableName()))); + } + + assertThat("Missing Variable:Observation combo", actualVariableObservation, containsInAnyOrder(expectedVariableObservation.toArray())); + } + + return ret; + } + + private Map assertValidPreviewRow(Map expected, JsonObject actual, Program program, List traits) { + Map ret = new HashMap<>(); + + assertNotNull(actual.get("trial")); + BrAPITrial trial = gson.fromJson(actual.getAsJsonObject("trial").getAsJsonObject("brAPIObject"), BrAPITrial.class); + ret.put("trial", trial); + + assertNotNull(actual.get("study")); + BrAPIStudy study = gson.fromJson(actual.getAsJsonObject("study").getAsJsonObject("brAPIObject"), BrAPIStudy.class); + ret.put("study", study); + + assertNotNull(actual.get("location")); + ProgramLocation location = gson.fromJson(actual.getAsJsonObject("location").getAsJsonObject("brAPIObject"), ProgramLocation.class); + ret.put("location", location); + + assertNotNull(actual.get("observationUnit")); + BrAPIObservationUnit ou = gson.fromJson(actual.getAsJsonObject("observationUnit").getAsJsonObject("brAPIObject"), BrAPIObservationUnit.class); + ret.put("observationUnit", ou); + + assertNotNull(actual.get("germplasm")); + BrAPIGermplasm germplasm = gson.fromJson(actual.getAsJsonObject("germplasm").getAsJsonObject("brAPIObject"), BrAPIGermplasm.class); + ret.put("germplasm", germplasm); + + List observations = null; + if(traits != null) { + assertNotNull(actual.get("observations")); + observations = gson.fromJson(actual.get("observations"), new TypeToken>(){}.getType()); + ret.put("observations", observations); + } + + assertNotNull(germplasm.getGermplasmName()); + assertEquals(expected.get(Columns.GERMPLASM_GID), germplasm.getAccessionNumber()); + if(expected.containsKey(Columns.TEST_CHECK) && StringUtils.isNotBlank((String)expected.get(Columns.TEST_CHECK))) { + assertEquals(expected.get(Columns.TEST_CHECK), + ou.getObservationUnitPosition() + .getEntryType() + .name() + .substring(0, 1)); + } + assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); + assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); + assertEquals(expected.get(Columns.EXP_DESCRIPTION), trial.getTrialDescription()); + assertEquals(expected.get(Columns.EXP_UNIT), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString()); + assertEquals(expected.get(Columns.EXP_UNIT), ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + assertEquals(expected.get(Columns.EXP_TYPE), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); + assertEquals(expected.get(Columns.EXP_TYPE), study.getStudyType()); + assertEquals(expected.get(Columns.ENV), Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); + assertEquals(expected.get(Columns.ENV_LOCATION), study.getLocationName()); + assertEquals(expected.get(Columns.ENV_LOCATION), location.getName()); + //TODO figure out how to get the actual season value +// assertEquals(expected.getInt(Columns.ENV_YEAR), Integer.parseInt(study.getSeasons().get(0))); + assertEquals(expected.get(Columns.EXP_UNIT_ID), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + + BrAPIObservationUnitLevelRelationship rep = null; + BrAPIObservationUnitLevelRelationship block = null; + for (BrAPIObservationUnitLevelRelationship rel : ou.getObservationUnitPosition().getObservationLevelRelationships()) { + if ("rep".equals(rel.getLevelName()) && rep == null) { + rep = rel; + } else if ("block".equals(rel.getLevelName()) && block == null) { + block = rel; + } + } + assertNotNull(rep); + assertNotNull(block); + assertEquals(expected.get(Columns.REP_NUM), rep.getLevelCode()); + assertEquals(expected.get(Columns.BLOCK_NUM), block.getLevelCode()); + if(expected.containsKey(Columns.ROW)) { + assertEquals(expected.get(Columns.ROW), ou.getObservationUnitPosition().getPositionCoordinateX()); + assertEquals(BrAPIPositionCoordinateTypeEnum.GRID_ROW, ou.getObservationUnitPosition().getPositionCoordinateXType()); + } + if(expected.containsKey(Columns.COLUMN)) { + assertEquals(expected.get(Columns.COLUMN), ou.getObservationUnitPosition().getPositionCoordinateY()); + assertEquals(BrAPIPositionCoordinateTypeEnum.GRID_COL, ou.getObservationUnitPosition().getPositionCoordinateYType()); + } + if(expected.containsKey(Columns.TREATMENT_FACTORS) && StringUtils.isNotBlank((String)expected.get(Columns.TREATMENT_FACTORS))) { + assertEquals(expected.get(Columns.TREATMENT_FACTORS), ou.getTreatments().get(0).getFactor()); + } + + if(traits != null) { + List expectedVariableObservation = new ArrayList<>(); + List actualVariableObservation = new ArrayList<>(); + observations.forEach(observation -> actualVariableObservation.add(String.format("%s:%s", Utilities.removeProgramKey(observation.getObservationVariableName(), program.getKey()), observation.getValue()))); + for(Trait trait : traits) { + expectedVariableObservation.add(String.format("%s:%s", trait.getObservationVariableName(), expected.get(trait.getObservationVariableName()))); + } + + assertThat("Missing Variable:Observation combo", actualVariableObservation, containsInAnyOrder(expectedVariableObservation.toArray())); + } + + return ret; + } + + private Program createProgram(String name, String abbv, String key, String referenceSource, List germplasm, List traits) throws ApiException, DoesNotExistException, ValidatorException, BadRequestException { + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + ProgramRequest programRequest1 = ProgramRequest.builder() + .name(name) + .abbreviation(abbv) + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key(key) + .build(); + + + TestUtils.insertAndFetchTestProgram(gson, client, programRequest1); + + // Get main program + Program program = programService.getByKey(key).get(); + + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId().toString()); + + if(germplasm != null && !germplasm.isEmpty()) { + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", referenceSource)); + newReference.setReferenceID(program.getId().toString()); + + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); + + germplasmDAO.importBrAPIGermplasm(germplasm, program.getId(), null); + } + + if(traits != null && !traits.isEmpty()) { + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + try { + ontologyService.createTraits(program.getId(), traits, user, false); + } catch (ValidatorException e) { + System.err.println(e.getErrors()); + throw e; + } + } + + return program; + } + + private List createGermplasm(int numToCreate) { + List germplasm = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String gid = ""+(i+1); + BrAPIGermplasm testGermplasm = new BrAPIGermplasm(); + testGermplasm.setGermplasmName(String.format("Germplasm %s [TEST-%s]", gid, gid)); + testGermplasm.setSeedSource("Wild"); + testGermplasm.setAccessionNumber(gid); + testGermplasm.setDefaultDisplayName(String.format("Germplasm %s", gid)); + JsonObject additionalInfo = new JsonObject(); + additionalInfo.addProperty("importEntryNumber", gid); + additionalInfo.addProperty("breedingMethod", "Allopolyploid"); + testGermplasm.setAdditionalInfo(additionalInfo); + List externalRef = new ArrayList<>(); + BrAPIExternalReference testReference = new BrAPIExternalReference(); + testReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); + testReference.setReferenceID(UUID.randomUUID().toString()); + externalRef.add(testReference); + testGermplasm.setExternalReferences(externalRef); + germplasm.add(testGermplasm); + } + + return germplasm; + } + + private List createTraits(int numToCreate) { + List traits = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String varName = "tt_test_" + (i + 1); + traits.add(Trait.builder() + .observationVariableName(varName) + .entity("Plant " + i) + .attribute("height " + i) + .traitDescription("test") + .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build()) + .scale(Scale.builder() + .scaleName("test scale") + .dataType(DataType.NUMERICAL) + .validValueMin(0) + .validValueMax(100) + .build()) + .method(Method.builder() + .description("test method") + .methodClass("test method") + .build()) + .build()); + } + + return traits; + } + + private JsonObject uploadAndVerifyFailure(Program program, File file, String expectedColumnError) throws InterruptedException, IOException { + Flowable> call = importTestUtils.uploadDataFile(file, null, true, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); + assertEquals(1, rowErrors.size()); + JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); + assertEquals(1, fieldErrors.size()); + JsonObject error = fieldErrors.get(0).getAsJsonObject(); + assertEquals(expectedColumnError, error.get("field").getAsString()); + assertEquals(422, error.get("httpStatusCode").getAsInt()); + + return result; + } + + private File writeDataToFile(List> data, List traits) throws IOException { + File file = File.createTempFile("test", ".csv"); + + List columns = new ArrayList<>(); + columns.add(Column.builder().value(Columns.GERMPLASM_NAME).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.GERMPLASM_GID).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.TEST_CHECK).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.EXP_TITLE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.EXP_DESCRIPTION).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.EXP_UNIT).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.EXP_TYPE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.ENV).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.ENV_LOCATION).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.ENV_YEAR).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.EXP_UNIT_ID).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.REP_NUM).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.BLOCK_NUM).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.ROW).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.COLUMN).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.TREATMENT_FACTORS).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.OBS_UNIT_ID).dataType(Column.ColumnDataType.STRING).build()); + + if(traits != null) { + traits.forEach(trait -> { + columns.add(Column.builder().value(trait.getObservationVariableName()).dataType(Column.ColumnDataType.STRING).build()); + }); + } + + ByteArrayOutputStream byteArrayOutputStream = CSVWriter.writeToCSV(columns, data); + FileOutputStream fos = new FileOutputStream(file); + fos.write(byteArrayOutputStream.toByteArray()); + + return file; + } + +} diff --git a/src/test/java/org/breedinginsight/brapps/importer/GermplasmTemplateMap.java b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java similarity index 80% rename from src/test/java/org/breedinginsight/brapps/importer/GermplasmTemplateMap.java rename to src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java index ade545d08..b52dbde0a 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/GermplasmTemplateMap.java +++ b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java @@ -1,7 +1,6 @@ package org.breedinginsight.brapps.importer; import com.google.gson.*; -import io.kowalski.fannypack.FannyPack; import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; @@ -31,7 +30,6 @@ import org.breedinginsight.daos.BreedingMethodDAO; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.Program; -import org.breedinginsight.model.Species; import org.breedinginsight.services.SpeciesService; import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; @@ -48,17 +46,17 @@ import static io.micronaut.http.HttpRequest.GET; import static io.micronaut.http.HttpRequest.POST; -import static io.micronaut.http.HttpRequest.PUT; import static org.junit.jupiter.api.Assertions.*; @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) -public class GermplasmTemplateMap extends BrAPITest { +public class GermplasmFileImportTest extends BrAPITest { - private FannyPack fp; + private static final String GERM_LIST_NAME = "germplasmListName"; + private static final String GERM_LIST_DESC = "germplasmListDescription"; private Program validProgram; - private String germplasmImportId; + private String germplasmMappingId; private BiUserEntity testUser; @Property(name = "brapi.server.reference-source") @@ -75,65 +73,23 @@ public class GermplasmTemplateMap extends BrAPITest { @Inject private DSLContext dsl; + private ImportTestUtils importTestUtils; + @Inject @Client("/${micronaut.bi.api.version}") RxHttpClient client; - private Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) + private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) (json, type, context) -> OffsetDateTime.parse(json.getAsString())) - .create(); + .create(); @BeforeAll public void setup() { - fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); - var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); - var brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); - - // Set up BrAPI - super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); - - // Species - Species validSpecies = speciesService.getAll().get(0); - SpeciesRequest speciesRequest = SpeciesRequest.builder() - .commonName(validSpecies.getCommonName()) - .id(validSpecies.getId()) - .build(); - - // Insert program - ProgramRequest program = ProgramRequest.builder() - .name("Test Program") - .species(speciesRequest) - .key("TEST") - .build(); - validProgram = insertAndFetchTestProgram(program); - - // Get germplasm system import - Flowable> call = client.exchange( - GET("/import/mappings?importName=germplasmtemplatemap").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - HttpResponse response = call.blockingFirst(); - germplasmImportId = JsonParser.parseString(response.body()).getAsJsonObject() - .getAsJsonObject("result") - .getAsJsonArray("data") - .get(0).getAsJsonObject().get("id").getAsString(); - - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); - dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), validProgram.getId()); - dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); - } - - public Program insertAndFetchTestProgram(ProgramRequest programRequest) { - - Flowable> call = client.exchange( - POST("/programs/", gson.toJson(programRequest)) - .contentType(MediaType.APPLICATION_JSON) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - - HttpResponse response = call.blockingFirst(); - JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); - Program program = gson.fromJson(result, Program.class); - return program; + importTestUtils = new ImportTestUtils(); + Map setupObjects = importTestUtils.setup(client, gson, dsl, speciesService, userDAO, super.getBrapiDsl(), "GermplasmTemplateMap"); + validProgram = (Program) setupObjects.get("program"); + germplasmMappingId = (String) setupObjects.get("mappingId"); + testUser = (BiUserEntity) setupObjects.get("testUser"); } // Tests @@ -164,13 +120,12 @@ public Program insertAndFetchTestProgram(ProgramRequest programRequest) { public void minimalImportUserSpecifiedEntryNumbersSuccess() { File file = new File("src/test/resources/files/germplasm_import/minimal_germplasm_import.csv"); String listName = "MinimalList"; - String listDescription = null; - Flowable> call = uploadDataFile(file, listName, listDescription, true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, listName), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); @@ -184,7 +139,7 @@ public void minimalImportUserSpecifiedEntryNumbersSuccess() { } // Check the germplasm list - checkGermplasmList(Germplasm.constructGermplasmListName(listName, validProgram), listDescription, germplasmNames); + checkGermplasmList(Germplasm.constructGermplasmListName(listName, validProgram), null, germplasmNames); } @Test @@ -196,12 +151,12 @@ public void fullImportPreviewSuccess() { String listName = "FullList"; String listDescription = "A full import"; - Flowable> call = uploadDataFile(file, listName, listDescription, false); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, listName, GERM_LIST_DESC, listDescription), false, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); @@ -217,9 +172,9 @@ public void fullImportPreviewSuccess() { assertEquals(fileData.getString(i, "Name"), germplasm.get("germplasmName").getAsString()); JsonObject additionalInfo = germplasm.getAsJsonObject("additionalInfo"); // Created Date (not present) - assertTrue(!additionalInfo.has(BrAPIAdditionalInfoFields.CREATED_DATE), "createdDate is present, but should not be"); + assertFalse(additionalInfo.has(BrAPIAdditionalInfoFields.CREATED_DATE), "createdDate is present, but should not be"); // Accession Number (not present) - assertTrue(!germplasm.has("accessionNumber"), "accessionNumber is present, but should not be"); + assertFalse(germplasm.has("accessionNumber"), "accessionNumber is present, but should not be"); // Pedigree (display names) String pedigree = germplasm.get("pedigree").getAsString(); String mother = !pedigree.isBlank() ? pedigree.split("/")[0] : null; @@ -235,7 +190,7 @@ public void fullImportPreviewSuccess() { JsonArray externalReferences = germplasm.getAsJsonArray("externalReferences"); for (JsonElement reference: externalReferences) { String referenceSource = reference.getAsJsonObject().get("referenceSource").getAsString(); - assertTrue(referenceSource != BRAPI_REFERENCE_SOURCE, "Germplasm UUID was present, but should not be"); + assertNotSame(referenceSource, BRAPI_REFERENCE_SOURCE, "Germplasm UUID was present, but should not be"); } // Synonyms @@ -256,12 +211,12 @@ public void fullImportCommitSuccess() { String listName = "FullList"; String listDescription = "A full import"; - Flowable> call = uploadDataFile(file, listName, listDescription, true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, listName, GERM_LIST_DESC, listDescription), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); @@ -297,7 +252,7 @@ public void fullImportCommitSuccess() { // External Reference germplasm JsonArray externalReferences = germplasm.getAsJsonArray("externalReferences"); - Boolean referenceFound = false; + boolean referenceFound = false; for (JsonElement reference: externalReferences) { String referenceSource = reference.getAsJsonObject().get("referenceSource").getAsString(); if (referenceSource.equals(BRAPI_REFERENCE_SOURCE)) { @@ -330,12 +285,12 @@ public void duplicateNameMarksDuplicates() { String listName = "DupNamesList"; String listDescription = "A full import"; - Flowable> call = uploadDataFile(file, listName, listDescription, false); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, listName, GERM_LIST_DESC, listDescription), false, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); @@ -357,12 +312,12 @@ public void duplicateNameMarksDuplicates() { public void OnlyMaleParentPreviewSuccess() { File file = new File("src/test/resources/files/germplasm_import/no_female_parent_blank_pedigree.csv"); - Flowable> call = uploadDataFile(file, "NoFemaleParentList", null, true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "NoFemaleParentList"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); @@ -378,7 +333,7 @@ public void OnlyMaleParentPreviewSuccess() { @SneakyThrows public void missingRequiredUserInput() { File file = new File("src/test/resources/files/germplasm_import/female_dbid_not_exist.csv"); - Flowable> call = uploadDataFile(file, null, null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(), true, client, validProgram, germplasmMappingId); HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { HttpResponse response = call.blockingFirst(); }); @@ -394,11 +349,11 @@ public void germplasmListNameDuplicateError() { String listName = "FullList"; String listDescription = "A full import"; - Flowable> call = uploadDataFile(file, listName, listDescription, true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, listName, GERM_LIST_DESC, listDescription), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); assertEquals(GermplasmProcessor.listNameAlreadyExists, result.getAsJsonObject("progress").get("message").getAsString()); @@ -408,12 +363,12 @@ public void germplasmListNameDuplicateError() { @SneakyThrows public void femaleParentDbIdNotExistError() { File file = new File("src/test/resources/files/germplasm_import/female_dbid_not_exist.csv"); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); List missingDbIds = List.of("1000", "1001", "1002"); @@ -425,12 +380,12 @@ public void femaleParentDbIdNotExistError() { @SneakyThrows public void femaleParentEntryNumberNotExistError() { File file = new File("src/test/resources/files/germplasm_import/female_entry_number_not_exist.csv"); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); List missingDbIds = List.of("1", "2", "3"); @@ -442,12 +397,12 @@ public void femaleParentEntryNumberNotExistError() { @SneakyThrows public void maleParentDbIdNotExistError() { File file = new File("src/test/resources/files/germplasm_import/male_dbid_not_exist.csv"); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); List missingDbIds = List.of("100", "101", "102"); @@ -459,12 +414,12 @@ public void maleParentDbIdNotExistError() { @SneakyThrows public void maleParentEntryNumberNotExistError() { File file = new File("src/test/resources/files/germplasm_import/male_entry_number_not_exist.csv"); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); List missingDbIds = List.of("1", "2", "3"); @@ -476,12 +431,12 @@ public void maleParentEntryNumberNotExistError() { @SneakyThrows public void badBreedingMethods() { File file = new File("src/test/resources/files/germplasm_import/bad_breeding_methods.csv"); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); @@ -502,12 +457,12 @@ public void badBreedingMethods() { @SneakyThrows public void someEntryNumbersError() { File file = new File("src/test/resources/files/germplasm_import/some_entry_numbers.csv"); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); assertEquals(GermplasmProcessor.missingEntryNumbersMsg, result.getAsJsonObject("progress").get("message").getAsString()); @@ -517,12 +472,12 @@ public void someEntryNumbersError() { @SneakyThrows public void duplicateEntryNumbersError() { File file = new File("src/test/resources/files/germplasm_import/duplicate_entry_numbers.csv"); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); List dups = List.of("1", "3"); @@ -537,7 +492,7 @@ public void nonNumericEntryNumbersError() { MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); // Upload file - String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmImportId); + String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmMappingId); Flowable> call = client.exchange( POST(uploadUrl, requestBody) .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) @@ -558,7 +513,7 @@ public void emptyRequiredFieldsError() { MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); // Upload file - String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmImportId); + String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmMappingId); Flowable> call = client.exchange( POST(uploadUrl, requestBody) .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) @@ -571,11 +526,11 @@ public void emptyRequiredFieldsError() { assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, e.getStatus()); JsonArray rowErrors = JsonParser.parseString((String) e.getResponse().getBody().get()).getAsJsonObject().getAsJsonArray("rowErrors"); - assertTrue(rowErrors.size() == 2, "Wrong number of row errors returned"); + assertEquals(2, rowErrors.size(), "Wrong number of row errors returned"); JsonObject rowError1 = rowErrors.get(0).getAsJsonObject(); JsonArray errors = rowError1.getAsJsonArray("errors"); - assertTrue(errors.size() == 1, "Not enough errors were returned"); + assertEquals(1, errors.size(), "Not enough errors were returned"); JsonObject error = errors.get(0).getAsJsonObject(); assertEquals(422, error.get("httpStatusCode").getAsInt(), "Incorrect http status code"); assertEquals("Name", error.get("field").getAsString(), "Incorrect field name"); @@ -583,7 +538,7 @@ public void emptyRequiredFieldsError() { JsonObject rowError2 = rowErrors.get(1).getAsJsonObject(); JsonArray errors2 = rowError2.getAsJsonArray("errors"); - assertTrue(errors2.size() == 1, "Not enough errors were returned"); + assertEquals(1, errors2.size(), "Not enough errors were returned"); JsonObject error2 = errors2.get(0).getAsJsonObject(); assertEquals(422, error2.get("httpStatusCode").getAsInt(), "Incorrect http status code"); assertEquals("Source", error2.get("field").getAsString(), "Incorrect field name"); @@ -595,13 +550,12 @@ public void emptyRequiredFieldsError() { public void headerCaseInsensitive() { File file = new File("src/test/resources/files/germplasm_import/germplasm_column_casing.csv"); String listName = "CaseInsensitiveList"; - String listDescription = null; - Flowable> call = uploadDataFile(file, listName, listDescription, true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, listName), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); @@ -615,7 +569,7 @@ public void headerCaseInsensitive() { } // Check the germplasm list - checkGermplasmList(Germplasm.constructGermplasmListName(listName, validProgram), listDescription, germplasmNames); + checkGermplasmList(Germplasm.constructGermplasmListName(listName, validProgram), null, germplasmNames); } @Test @@ -625,7 +579,7 @@ public void missingRequiredFieldHeaderError() { MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); // Upload file - String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmImportId); + String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmMappingId); Flowable> call = client.exchange( POST(uploadUrl, requestBody) .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) @@ -646,7 +600,7 @@ public void missingOptionalFieldHeaderError() { MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); // Upload file - String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmImportId); + String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmMappingId); Flowable> call = client.exchange( POST(uploadUrl, requestBody) .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) @@ -665,12 +619,12 @@ public void missingOptionalFieldHeaderError() { public void circularParentDependencyError() { File file = new File("src/test/resources/files/germplasm_import/circular_parent_dependencies.csv"); MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); assertEquals(GermplasmProcessor.circularDependency, result.getAsJsonObject("progress").get("message").getAsString()); @@ -681,62 +635,17 @@ public void circularParentDependencyError() { public void selfReferenceParentError() { File file = new File("src/test/resources/files/germplasm_import/self_ref_parent_dependencies.csv"); MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); assertEquals(GermplasmProcessor.circularDependency, result.getAsJsonObject("progress").get("message").getAsString()); } - public Flowable> uploadDataFile(File file, String listName, String listDescription, Boolean commit) { - MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); - - // Upload file - String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmImportId); - Flowable> call = client.exchange( - POST(uploadUrl, requestBody) - .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.OK, response.getStatus()); - JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); - String importId = result.get("importId").getAsString(); - - // Process data - String url = String.format("/programs/%s/import/mappings/%s/data/%s/%s", validProgram.getId(), germplasmImportId, importId, commit ? "commit" : "preview"); - Map listBody = new HashMap<>(); - listBody.put("germplasmListName", listName); - listBody.put("germplasmListDescription", listDescription); - Flowable> processCall = client.exchange( - PUT(url, listBody) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - return processCall; - } - - public HttpResponse getUploadedFile(String importId) throws InterruptedException { - Flowable> call = client.exchange( - GET(String.format("/programs/%s/import/mappings/%s/data/%s?mapping=true", validProgram.getId(), germplasmImportId, importId)) - .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - HttpResponse response = call.blockingFirst(); - - if (response.getStatus().equals(HttpStatus.ACCEPTED)) { - Thread.sleep(1000); - return getUploadedFile(importId); - } else { - return response; - } - - - } - public void checkBasicResponse(JsonObject germplasm, Table fileData, Integer i) { @@ -784,8 +693,8 @@ public void checkGermplasmList(String listName, String listDescription, List response = call.blockingFirst(); JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); JsonArray data = result.getAsJsonArray("data"); - Boolean nameFound = false; - Boolean descriptionFound = false; + boolean nameFound = false; + boolean descriptionFound = false; String listId = null; for (JsonElement listElement: data) { JsonObject listObject = listElement.getAsJsonObject(); @@ -827,9 +736,7 @@ public void checkGermplasmList(String listName, String listDescription, List> call = client.exchange( + POST("/programs/", gson.toJson(programRequest)) + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpResponse response = call.blockingFirst(); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + return gson.fromJson(result, Program.class); + } + public Flowable> uploadDataFile(File file, Map userData, Boolean commit, RxHttpClient client, Program program, String mappingId) { + MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); + + // Upload file + String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", program.getId(), mappingId); + Flowable> call = client.exchange( + POST(uploadUrl, requestBody) + .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + String importId = result.get("importId").getAsString(); + + // Process data + String url = String.format("/programs/%s/import/mappings/%s/data/%s/%s", program.getId(), mappingId, importId, commit ? "commit" : "preview"); + Flowable> processCall = client.exchange( + PUT(url, userData) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + return processCall; + } + + public HttpResponse getUploadedFile(String importId, RxHttpClient client, Program program, String mappingId) throws InterruptedException { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/import/mappings/%s/data/%s?mapping=true", program.getId(), mappingId, importId)) + .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + + if (response.getStatus().equals(HttpStatus.ACCEPTED)) { + Thread.sleep(1000); + return getUploadedFile(importId, client, program, mappingId); + } else { + return response; + } + } + + public Map setup(RxHttpClient client, Gson gson, DSLContext dsl, SpeciesService speciesService, UserDAO userDAO, DSLContext brapiDsl, String mappingTemplateName) { + var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + var brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); + + // Set up BrAPI + brapiDsl.execute(brapiFp.get("InsertSpecies")); + + // Species + Species validSpecies = speciesService.getAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + + // Insert program + ProgramRequest program = ProgramRequest.builder() + .name("Test Program") + .species(speciesRequest) + .key("TEST") + .build(); + Program validProgram = this.insertAndFetchTestProgram(program, client, gson); + + // Get import + Flowable> call = client.exchange( + GET("/import/mappings?importName="+mappingTemplateName).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + String mappingId = JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + BiUserEntity testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), validProgram.getId()); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + return Map.of("program", validProgram, + "mappingId", mappingId, + "testUser", testUser, + "securityFp", securityFp); + } + + + + public JsonObject uploadAndFetch(File file, Map userData, Boolean commit, RxHttpClient client, Program program, String mappingId) throws InterruptedException { + Flowable> call = uploadDataFile(file, userData, commit, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + return result; + } +} diff --git a/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java b/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java index 15c3e33be..2abe8e486 100644 --- a/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java +++ b/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java @@ -17,6 +17,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import java.lang.reflect.Field; +import java.time.Duration; +import java.time.temporal.ChronoUnit; import java.util.*; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -62,7 +64,7 @@ public ApiResponse, Optional locations; private FannyPack fp; @@ -74,11 +81,12 @@ public class ResponseUtilsIntegrationTest extends DatabaseTest { private PlaceDao locationDao; @Inject private UserDAO userDAO; + @Inject + private ProgramLocationService locationService; // Set up program locations @BeforeAll - @SneakyThrows - public void setup() { + public void setup() throws MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException, NoSuchMethodException, InvocationTargetException, IllegalAccessException { // Insert our traits into the db fp = FannyPack.fill("src/test/resources/sql/ResponseUtilsIntegrationTest.sql"); @@ -90,12 +98,37 @@ public void setup() { // Insert program dsl.execute(fp.get("InsertProgram")); + validProgram = new Program(programDao.findAll().get(0)); // Insert program locations - dsl.execute(fp.get("InsertProgramLocations")); - - validProgram = programDao.findAll().get(0); - + List newLocations = new ArrayList<>(); + newLocations.add(ProgramLocationRequest.builder() + .name("place1") + .abbreviation("abbrev1") + .slope(new BigDecimal("1.1")) + .build()); + for(int i = 2; i < 25; i++) { + newLocations.add(ProgramLocationRequest.builder() + .name("place"+i) + .abbreviation("abbrev"+i) + .slope(new BigDecimal(((Math.random()*10)%2 == 0 ? "" : "-")+Math.random()*20.0)) + .build()); + } + for(int i = 25; i < 31; i++) { + newLocations.add(ProgramLocationRequest.builder() + .name("place"+i) + .build()); + } + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + + //accessing the private method to bypass the fetch of a program from the db/brapi (which fails bc the program isn't in the brapi server) + Method createLocationMethod = locationService.getClass() + .getDeclaredMethod("createLocation", AuthenticatedUser.class, Program.class, ProgramLocationRequest.class); + createLocationMethod.setAccessible(true); + for (ProgramLocationRequest location : newLocations) { + createLocationMethod.invoke(locationService, user, validProgram, location); + } + createLocationMethod.setAccessible(false); locations = locationDao.findAll(); }