diff --git a/.env.template b/.env.template index 0f100a7db..407b99d50 100644 --- a/.env.template +++ b/.env.template @@ -42,9 +42,11 @@ BRAPI_REFERENCE_SOURCE=breedinginsight.org WEB_BASE_URL=http://localhost:8080 # Email server -EMAIL_RELAY_HOST=mailhog -EMAIL_RELAY_PORT=1025 -EMAIL_FROM=bidevteam@cornell.edu +EMAIL_RELAY_HOST= +EMAIL_RELAY_PORT=<1025 for development, 2587 for production> +EMAIL_FROM=noreply@breedinginsight.org +#EMAIL_RELAY_LOGIN= +#EMAIL_RELAY_PASSWORD= GIGWA_HOST= GIGWA_USER= diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java new file mode 100644 index 000000000..51799b346 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIPedigreeController.java @@ -0,0 +1,137 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2; + +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.annotation.*; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIIndexPagination; +import org.brapi.v2.model.BrAPIMetadata; +import org.brapi.v2.model.germ.BrAPIPedigreeNode; +import org.brapi.v2.model.germ.response.BrAPIPedigreeListResponse; +import org.brapi.v2.model.germ.response.BrAPIPedigreeListResponseResult; +import org.breedinginsight.api.auth.ProgramSecured; +import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; +import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import org.breedinginsight.brapi.v2.dao.BrAPIPedigreeDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.utilities.Utilities; + +import javax.annotation.Nullable; +import javax.inject.Inject; +import java.util.*; + +@Slf4j +@Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) +@Secured(SecurityRule.IS_AUTHENTICATED) +public class BrAPIPedigreeController { + private final BrAPIPedigreeDAO pedigreeDAO; + + private final ProgramService programService; + + @Inject + public BrAPIPedigreeController(BrAPIPedigreeDAO pedigreeDAO, + ProgramService programService) { + this.pedigreeDAO = pedigreeDAO; + this.programService = programService; + } + + @Get("/pedigree") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse pedigreeGet(@PathVariable("programId") UUID programId, + @Nullable @QueryValue("accessionNumber") String accessionNumber, + @Nullable @QueryValue("collection") String collection, + @Nullable @QueryValue("familyCode") String familyCode, + @Nullable @QueryValue("binomialName") String binomialName, + @Nullable @QueryValue("genus") Boolean genus, + @Nullable @QueryValue("species") String species, + @Nullable @QueryValue("synonym") String synonym, + @Nullable @QueryValue("includeParents") Boolean includeParents, + @Nullable @QueryValue("includeSiblings") Boolean includeSiblings, + @Nullable @QueryValue("includeProgeny") Boolean includeProgeny, + @Nullable @QueryValue("includeFullTree") Boolean includeFullTree, + @Nullable @QueryValue("pedigreeDepth") Integer pedigreeDepth, + @Nullable @QueryValue("progenyDepth") Integer progenyDepth, + @Nullable @QueryValue("commonCropName") String commonCropName, + @Nullable @QueryValue("programDbId") String programDbId, + @Nullable @QueryValue("trialDbId") String trialDbId, + @Nullable @QueryValue("studyDbId") String studyDbId, + @Nullable @QueryValue("germplasmDbId") String germplasmDbId, + @Nullable @QueryValue("germplasmName") String germplasmName, + @Nullable @QueryValue("germplasmPUI") String germplasmPUI, + @Nullable @QueryValue("externalReferenceId") String externalReferenceId, + @Nullable @QueryValue("externalReferenceSource") String externalReferenceSource, + @Nullable @QueryValue("page") Integer page, + @Nullable @QueryValue("pageSize") Integer pageSize) { + + log.debug("pedigreeGet: fetching pedigree by filters"); + + Optional program = programService.getById(programId); + if(program.isEmpty()) { + log.warn("Program id: " + programId + " not found"); + return HttpResponse.notFound(); + } + + try { + List pedigree = pedigreeDAO.getPedigree( + program.get(), + Optional.ofNullable(includeParents), + Optional.ofNullable(includeSiblings), + Optional.ofNullable(includeProgeny), + Optional.ofNullable(includeFullTree), + Optional.ofNullable(pedigreeDepth), + Optional.ofNullable(progenyDepth), + Optional.ofNullable(germplasmName)); + + return HttpResponse.ok( + new BrAPIPedigreeListResponse() + .metadata(new BrAPIMetadata().pagination(new BrAPIIndexPagination().currentPage(0) + .totalPages(1) + .pageSize(pedigree.size()) + .totalCount(pedigree.size()))) + .result(new BrAPIPedigreeListResponseResult().data(pedigree)) + ); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "error fetching pedigree"); + } + } + + @Post("/pedigree") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse pedigreePost(@PathVariable("programId") UUID programId, @Body List body) { + //DO NOT IMPLEMENT - Users are only able to create pedigree via the DeltaBreed UI + return HttpResponse.notFound(); + } + + @Put("/pedigree") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse pedigreePut(@PathVariable("programId") UUID programId, @Body Map body) { + //DO NOT IMPLEMENT - Users aren't yet able to update observation units + return HttpResponse.notFound(); + } + + // TODO: search and retrieve endpoints + // already have some work done, search call in BrAPIPedigreeDAO + +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java index d6e4dde43..b5249516e 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java @@ -43,6 +43,7 @@ import java.io.IOException; import java.util.List; import java.util.UUID; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Slf4j @@ -147,7 +148,7 @@ public BrAPIServerInfoResponse programServerinfo(@PathVariable("programId") UUID .build(); for(BrAPIService service : programServices) { - service.setService(programBrAPIBase + service.getService()); + service.setService(service.getService()); } BrAPIServerInfo programServerInfo = new BrAPIServerInfo(); @@ -230,7 +231,10 @@ private HttpResponse executeRequest(String path, UUID programId, HttpReq } private HttpResponse makeCall(Request brapiRequest) { - OkHttpClient client = new OkHttpClient(); + // TODO: use config parameter for timeout + OkHttpClient client = new OkHttpClient.Builder() + .readTimeout(5, TimeUnit.MINUTES) + .build(); try (Response brapiResponse = client.newCall(brapiRequest).execute()) { if(brapiResponse.isSuccessful()) { try(ResponseBody body = brapiResponse.body()) { diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java new file mode 100644 index 000000000..a00957f2c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java @@ -0,0 +1,215 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2.dao; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.ApiResponse; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.model.queryParams.germplasm.PedigreeQueryParams; +import org.brapi.client.v2.modules.germplasm.PedigreeApi; +import org.brapi.v2.model.germ.BrAPIPedigreeNode; +import org.brapi.v2.model.germ.request.BrAPIPedigreeSearchRequest; +import org.brapi.v2.model.germ.response.BrAPIPedigreeListResponse; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; + +@Singleton +@Slf4j +public class BrAPIPedigreeDAO { + + private ProgramDAO programDAO; + private final BrAPIDAOUtil brAPIDAOUtil; + private final BrAPIEndpointProvider brAPIEndpointProvider; + private final String referenceSource; + + @Inject + public BrAPIPedigreeDAO(ProgramDAO programDAO, BrAPIDAOUtil brAPIDAOUtil, + BrAPIEndpointProvider brAPIEndpointProvider, + @Property(name = "brapi.server.reference-source") String referenceSource) { + this.programDAO = programDAO; + this.brAPIDAOUtil = brAPIDAOUtil; + this.brAPIEndpointProvider = brAPIEndpointProvider; + this.referenceSource = referenceSource; + } + + /** + * Retrieves the pedigree of a given program and optional filters. Used by Helium. TODO: Add rest of parameters + * + * @param program The program for which the pedigree is requested. + * @param includeParents Flag to indicate whether to include parent nodes in the pedigree. (optional) + * @param includeSiblings Flag to indicate whether to include sibling nodes in the pedigree. (optional) + * @param includeProgeny Flag to indicate whether to include progeny nodes in the pedigree. (optional) + * @param includeFullTree Flag to indicate whether to include the full pedigree tree or only immediate ancestors and descendants. (optional) + * @param pedigreeDepth The maximum depth of ancestors and descendants to include in the pedigree. (optional) + * @param progenyDepth The maximum depth of progeny to include in the pedigree. (optional) + * @param germplasmName The name of the germplasm to which the pedigree is limited. (optional) + * @return A list of pedigree nodes representing the pedigree of the program. + * @throws ApiException If an error occurs while making the BrAPI call. + */ + public List getPedigree( + Program program, + Optional includeParents, + Optional includeSiblings, + Optional includeProgeny, + Optional includeFullTree, + Optional pedigreeDepth, + Optional progenyDepth, + Optional germplasmName + ) throws ApiException { + + PedigreeQueryParams pedigreeRequest = new PedigreeQueryParams(); + + // TODO: Issue with BrAPI server programDbId filtering, think germplasm are linked to program through observation + // units and doesn't work if don't have any loaded + // use external refs instead for now + //pedigreeSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + + String extRefId = program.getId().toString(); + String extRefSrc = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS); + pedigreeRequest.externalReferenceId(extRefId); + pedigreeRequest.externalReferenceSource(extRefSrc); + + includeParents.ifPresent(pedigreeRequest::includeParents); + includeSiblings.ifPresent(pedigreeRequest::includeSiblings); + includeProgeny.ifPresent(pedigreeRequest::includeProgeny); + includeFullTree.ifPresent(pedigreeRequest::includeFullTree); + pedigreeDepth.ifPresent(pedigreeRequest::pedigreeDepth); + progenyDepth.ifPresent(pedigreeRequest::progenyDepth); + germplasmName.ifPresent(pedigreeRequest::germplasmName); + // TODO: other parameters + + // TODO: write utility to do paging instead of hardcoding + pedigreeRequest.pageSize(100000); + + ApiResponse brapiPedigree; + try { + brapiPedigree = brAPIEndpointProvider + .get(programDAO.getCoreClient(program.getId()), PedigreeApi.class) + .pedigreeGet(pedigreeRequest); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Error making BrAPI call", e); + } + + List pedigreeNodes = brapiPedigree.getBody().getResult().getData(); + // TODO: once Helium is constructing nodes from DbId we can strip program keys but won't in the mean time + //stripProgramKeys(pedigreeNodes, program.getKey()); + return pedigreeNodes; + } + + /** + * Searches for pedigree nodes based on the given parameters. Not used by Helium but keeping commented out for + * now in case we want to implement the search endpoints in the future, work here has already been started to + * support that. + * + * TODO: Add rest of parameters + * + * @param program The program to search for pedigree nodes. + * @param includeParents Optional boolean to include parents in the search. + * @param includeSiblings Optional boolean to include siblings in the search. + * @param includeProgeny Optional boolean to include progeny in the search. + * @param includeFullTree Optional boolean to include the full pedigree tree in the search. + * @param pedigreeDepth Optional integer for the maximum depth of the pedigree tree. + * @param progenyDepth Optional integer for the maximum depth of the progeny tree. + * @param germplasmName Optional String to filter the search by germplasm name. + * @return A List of BrAPIPedigreeNode objects that match the search criteria. + * @throws ApiException If an error occurs while searching for pedigree nodes. + */ + /* + public List searchPedigree(Program program, + Optional includeParents, + Optional includeSiblings, + Optional includeProgeny, + Optional includeFullTree, + Optional pedigreeDepth, + Optional progenyDepth, + Optional germplasmName + ) throws ApiException { + + BrAPIPedigreeSearchRequest pedigreeSearchRequest = new BrAPIPedigreeSearchRequest(); + // TODO: Issue with BrAPI server programDbId filtering, think germplasm are linked to program through observation + // units and doesn't work if don't have any loaded + // use external refs instead for now + //pedigreeSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + + // Just use program UUID, shouldn't have any collisions and don't want to get all the germplasm if we were also + // using source because search is OR rather than AND + String extRefId = program.getId().toString(); + pedigreeSearchRequest.addExternalReferenceIdsItem(extRefId); + + includeParents.ifPresent(pedigreeSearchRequest::includeParents); + includeSiblings.ifPresent(pedigreeSearchRequest::includeSiblings); + includeProgeny.ifPresent(pedigreeSearchRequest::includeProgeny); + includeFullTree.ifPresent(pedigreeSearchRequest::includeFullTree); + pedigreeDepth.ifPresent(pedigreeSearchRequest::setPedigreeDepth); + progenyDepth.ifPresent(pedigreeSearchRequest::setProgenyDepth); + germplasmName.ifPresent(pedigreeSearchRequest::addGermplasmNamesItem); + // TODO: other parameters + + PedigreeApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), PedigreeApi.class); + + List pedigreeNodes = brAPIDAOUtil.search( + api::searchPedigreePost, + api::searchPedigreeSearchResultsDbIdGet, + pedigreeSearchRequest + ); + + // TODO: once Helium is constructing nodes from DbId we can strip program keys but won't in the mean time + //stripProgramKeys(pedigreeNodes, program.getKey()); + + return pedigreeNodes; + } + */ + + /** + * Removes the program key from the germplasm names in the list of pedigree nodes. Not used currently but will be in + * future if we decide to strip the program keys when Helium has been updated to use the germplasmDbId for uniqueness + * rather than germplasmName. + * + * @param pedigreeNodes The list of pedigree nodes. + * @param programKey The program key to be removed. + */ + private void stripProgramKeys(List pedigreeNodes, String programKey) { + pedigreeNodes.forEach(node -> { + node.setGermplasmName(Utilities.removeProgramKeyAnyAccession(node.getGermplasmName(), programKey)); + // TODO: pedigree stripping not working right + //node.setPedigreeString(Utilities.removeProgramKeyAnyAccession(node.getPedigreeString(), programKey)); + if (node.getParents() != null) { + node.getParents().forEach(parent -> { + parent.setGermplasmName(Utilities.removeProgramKeyAnyAccession(parent.getGermplasmName(), programKey)); + }); + } + if (node.getProgeny() != null) { + node.getProgeny().forEach(progeny -> { + progeny.setGermplasmName(Utilities.removeProgramKeyAnyAccession(progeny.getGermplasmName(), programKey)); + }); + } + }); + } + +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index caab88879..dd9bde018 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,5 +1,8 @@ package org.breedinginsight.brapi.v2.services; +import com.github.filosganga.geogson.model.Coordinates; +import com.github.filosganga.geogson.model.positions.SinglePosition; +import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; import io.micronaut.http.MediaType; import io.micronaut.http.server.exceptions.InternalServerException; @@ -465,6 +468,26 @@ private Map createExportRow( row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); row.put(ExperimentObservation.Columns.ENV, Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); + + // Lat, Long, Elevation + Coordinates coordinates = extractCoordinates(ou); + Optional.ofNullable(coordinates) + .map(c -> doubleToString(c.getLat())) + .ifPresent(lat -> row.put(ExperimentObservation.Columns.LAT, lat)); + Optional.ofNullable(coordinates) + .map(c -> doubleToString(c.getLon())) + .ifPresent(lon -> row.put(ExperimentObservation.Columns.LONG, lon)); + Optional.ofNullable(coordinates) + .map(c -> doubleToString(c.getAlt())) + .ifPresent(elevation -> row.put(ExperimentObservation.Columns.ELEVATION, elevation)); + + // RTK + JsonObject additionalInfo = ou.getAdditionalInfo(); + String rtk = ( additionalInfo==null || additionalInfo.get(BrAPIAdditionalInfoFields.RTK) ==null ) + ? null + : additionalInfo.get(BrAPIAdditionalInfoFields.RTK).getAsString(); + row.put(ExperimentObservation.Columns.RTK, rtk); + BrAPISeason season = seasonDAO.getSeasonById(study.getSeasons().get(0), program.getId()); row.put(ExperimentObservation.Columns.ENV_YEAR, season.getYear()); row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); @@ -484,11 +507,13 @@ private Map createExportRow( .findFirst(); blockLevel.ifPresent(brAPIObservationUnitLevelRelationship -> row.put(ExperimentObservation.Columns.BLOCK_NUM, Integer.parseInt(brAPIObservationUnitLevelRelationship.getLevelCode()))); - if (ou.getObservationUnitPosition() != null && ou.getObservationUnitPosition().getPositionCoordinateX() != null && - ou.getObservationUnitPosition().getPositionCoordinateY() != null) { + + //Row and Column + if ( ou.getObservationUnitPosition() != null ) { row.put(ExperimentObservation.Columns.ROW, ou.getObservationUnitPosition().getPositionCoordinateX()); row.put(ExperimentObservation.Columns.COLUMN, ou.getObservationUnitPosition().getPositionCoordinateY()); } + if (ou.getTreatments() != null && !ou.getTreatments().isEmpty()) { row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, ou.getTreatments().get(0).getFactor()); } else { @@ -499,7 +524,25 @@ private Map createExportRow( return row; } - + private String doubleToString(double val){ + return Double.isNaN(val) ? null : String.valueOf( val ); + } + private Coordinates extractCoordinates(BrAPIObservationUnit ou){ + Coordinates coordinates = null; + if ( ou.getObservationUnitPosition()!=null + && ou.getObservationUnitPosition().getGeoCoordinates()!=null + && ou.getObservationUnitPosition().getGeoCoordinates().getGeometry()!=null + && ou.getObservationUnitPosition().getGeoCoordinates().getGeometry().positions()!=null + ) + { + Object o = ou.getObservationUnitPosition().getGeoCoordinates().getGeometry().positions(); + if (o instanceof SinglePosition){ + SinglePosition sp = (SinglePosition)o; + coordinates= sp.coordinates(); + } + } + return coordinates; + } private void addObsVarColumns( List columns, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 45d26f7cb..95e196c5b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -882,8 +882,19 @@ private void validateObservations(ValidationErrors validationErrors, // had been saved prior to import } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { + // different data means validations still need to happen + // TODO consider moving these two calls into a separate method since called twice together + validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); + + //Timestamp validation + if(timeStampColByPheno.containsKey(phenoCol.name())) { + Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); + validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); + } + // add a change log entry when updating the value of an observation - if (commit) { + // only will update and thereby need change log entry if no error + if (commit && (!validationErrors.hasErrors())) { BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); String timestamp = formatter.format(OffsetDateTime.now()); @@ -1175,11 +1186,21 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra } boolean isTimestampMatched(String observationHash, String timeStamp) { - OffsetDateTime priorStamp = existingObsByObsHash.get(observationHash).getObservationTimeStamp(); + OffsetDateTime priorStamp = null; + if(existingObsByObsHash.get(observationHash)!=null){ + priorStamp = existingObsByObsHash.get(observationHash).getObservationTimeStamp(); + } if (priorStamp == null) { return timeStamp == null; } - return priorStamp.isEqual(OffsetDateTime.parse(timeStamp)); + boolean isMatched = false; + try { + isMatched = priorStamp.isEqual(OffsetDateTime.parse(timeStamp)); + } catch(DateTimeParseException e){ + //if timestamp is invalid DateTime not equal to validated priorStamp + log.error(e.getMessage(), e); + } + return isMatched; } boolean isValueMatched(String observationHash, String value) { @@ -1223,13 +1244,14 @@ private void fetchOrCreateObservationPIO(Program program, } if (existingObsByObsHash.containsKey(key)) { - if (!isObservationMatched(key, value, column, rowNum)){ + // Update observation value only if it is changed and new value is not blank. + if (!isObservationMatched(key, value, column, rowNum) && StringUtils.isNotBlank(value)){ // prior observation with updated value newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); if (!isValueMatched(key, value)){ newObservation.setValue(value); - } else if (!isTimestampMatched(key, timeStampValue)) { + } else if (!StringUtils.isBlank(timeStampValue) && !isTimestampMatched(key, timeStampValue)) { DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timeStampValue)); newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); diff --git a/src/main/java/org/breedinginsight/services/UserService.java b/src/main/java/org/breedinginsight/services/UserService.java index ebc4e94b4..7502ff8b2 100644 --- a/src/main/java/org/breedinginsight/services/UserService.java +++ b/src/main/java/org/breedinginsight/services/UserService.java @@ -388,7 +388,7 @@ private void sendAccountSignUpEmail(BiUserEntity user, SignedJWT jwtToken) { emailTemplate.add("expiration_time", expirationTime); String filledBody = emailTemplate.render(); - String subject = "New Account Sign Up"; + String subject = "Activate DeltaBreed Account"; // Send email emailUtil.sendEmail(user.getEmail(), subject, filledBody); diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 2ff850d55..2231f3a16 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -86,6 +86,16 @@ public static String removeUnknownProgramKey(String original) { return original.replaceAll("\\[.*\\]", "").trim(); } + /** + * Removes the program key from a string with any accession number. + * + * @param str The string to remove the program key from + * @param programKey The program key to remove + * @return The modified string + */ + public static String removeProgramKeyAnyAccession(String str, String programKey) { + return str.replaceAll("\\[" + programKey + "-.*\\]", "").trim(); + } /** * Remove program key from a string. Returns a new value instead of altering original string. diff --git a/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java b/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java index f0f6652d9..a839715a1 100644 --- a/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java +++ b/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java @@ -19,12 +19,10 @@ import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.HttpServerException; +import org.apache.commons.lang3.StringUtils; import javax.inject.Singleton; -import javax.mail.Message; -import javax.mail.MessagingException; -import javax.mail.Session; -import javax.mail.Transport; +import javax.mail.*; import javax.mail.internet.InternetAddress; import javax.mail.internet.MimeMessage; import java.io.UnsupportedEncodingException; @@ -40,13 +38,29 @@ public class EmailUtil { private Integer smtpHostPort; @Property(name = "email.from") private String fromEmail; + @Property(name = "email.relay-server.login") + private String smtpLogin; + @Property(name = "email.relay-server.password") + private String smtpPassword; private Session getSmtpHost() { Properties props = new Properties(); props.put("mail.smtp.host", smtpHostServer); props.put("mail.smtp.port", smtpHostPort); props.put("mail.debug", true); - return Session.getInstance(props, null); + Authenticator auth = null; + if (StringUtils.isNotBlank(smtpLogin) && StringUtils.isNotBlank(smtpPassword)) { + props.put("mail.smtp.auth", true); + props.put("mail.smtp.ssl.trust", smtpHostServer); + props.put("mail.smtp.starttls.enable", true); + auth = new Authenticator() { + @Override + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(smtpLogin, smtpPassword); + } + }; + } + return Session.getInstance(props, auth); } public void sendEmail(String toEmail, String subject, String body){ diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index aec211b82..10acaf1f0 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -177,6 +177,8 @@ email: relay-server: host: ${EMAIL_RELAY_HOST} port: ${EMAIL_RELAY_PORT} + login: ${EMAIL_RELAY_LOGIN:null} + password: ${EMAIL_RELAY_PASSWORD:null} from: ${EMAIL_FROM} redisson: diff --git a/src/main/resources/email/newAccountEmail.st b/src/main/resources/email/newAccountEmail.st index a5fd73303..6fa9c1154 100644 --- a/src/main/resources/email/newAccountEmail.st +++ b/src/main/resources/email/newAccountEmail.st @@ -2,7 +2,7 @@ Welcome to Breeding Insight! We use a common login system with ORCID to provide authentication and account security. You will need a current ORCID iD and account to log in to Breeding Insight. If you do not already have an ORCID iD, you can create one here: https://orcid.org/register -To activate your Breeding Insight account and connect your ORCID iD to Breeding Insight, use this link: +To activate your DeltaBreed account and connect your ORCID iD to DeltaBreed, use this link: diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index e25a7e336..b8a9bf8f8 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0+723 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c936dc56136e0006616bb5d713434110100a4cb4 +version=v0.10.0+745 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6662651d581bb80396250f3ff38bc9443c4a4744 diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 6bc203d64..48bffd4c2 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -85,7 +85,6 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class ExperimentFileImportTest extends BrAPITest { - private static final String OVERWRITE = "overwrite"; private FannyPack securityFp; private String mappingId; @@ -822,6 +821,81 @@ public void importNewObservationDataByObsUnitId(boolean commit) { } } + /* + Scenario: + - an experiment was created with observations + - an overwrite operation is attempted with blank observation values + - verify blank observation values do not overwrite original values + */ + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void verifyBlankObsInOverwriteIsNoOp(boolean commit) { + List traits = importTestUtils.createTraits(1); + Program program = createProgram("Overwrite Attempt With Blank Obs"+(commit ? "C" : "P"), "NOOP"+(commit ? "C" : "P"), "NOOP"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), "1"); // Valid observation value. + + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + + // Fetch the ObsUnitId to use in the overwrite upload. + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); + Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); + assertTrue(trialIdXref.isPresent()); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); + Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); + assertTrue(ouIdXref.isPresent()); + + assertRowSaved(newExp, program, traits); + + Map newObsVar = new HashMap<>(); + newObsVar.put(Columns.GERMPLASM_GID, "1"); + newObsVar.put(Columns.TEST_CHECK, "T"); + newObsVar.put(Columns.EXP_TITLE, "Test Exp"); + newObsVar.put(Columns.EXP_UNIT, "Plot"); + newObsVar.put(Columns.EXP_TYPE, "Phenotyping"); + newObsVar.put(Columns.ENV, "New Env"); + newObsVar.put(Columns.ENV_LOCATION, "Location A"); + newObsVar.put(Columns.ENV_YEAR, "2023"); + newObsVar.put(Columns.EXP_UNIT_ID, "a-1"); + newObsVar.put(Columns.REP_NUM, "1"); + newObsVar.put(Columns.BLOCK_NUM, "1"); + newObsVar.put(Columns.ROW, "1"); + newObsVar.put(Columns.COLUMN, "1"); + newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); // Indicates this is an overwrite. + newObsVar.put(traits.get(0).getObservationVariableName(), ""); // Empty string should be no op. + + Map requestBody = new HashMap<>(); + requestBody.put("overwrite", "true"); + requestBody.put("overwriteReason", "testing"); + + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), requestBody, commit, client, program, mappingId); + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + // Verify that the overwrite attempt with blank observation value did not overwrite the original value. + assertRowSaved(newExp, program, traits); + if(commit) { + assertRowSaved(newExp, program, traits); + } else { + assertValidPreviewRow(newExp, row, program, traits); + } + } @ParameterizedTest @ValueSource(booleans = {true, false}) @@ -1086,15 +1160,15 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { /* Scenario: - - an experiment was created with observations - - do a second upload with additional observations for the experiment. Make sure the original observations are blank values - - verify the second set of observations get uploaded successfully + - Create an experiment with valid observations. + - Upload a second file with (1) a blank observation, (2) a changed valid observation, and (3) a new observation for the experiment. + - Verify that (1) the blank observation makes no change, (2) the changed observation is overwritten, and (3) new observations are appended to the experiment. */ @ParameterizedTest @ValueSource(booleans = {true, false}) @SneakyThrows public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { - List traits = importTestUtils.createTraits(2); + List traits = importTestUtils.createTraits(3); Program program = createProgram("Exp with additional Uploads (blank) "+(commit ? "C" : "P"), "EXAUB"+(commit ? "C" : "P"), "EXAUB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); @@ -1110,7 +1184,9 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newExp.put(Columns.BLOCK_NUM, "1"); newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - newExp.put(traits.get(0).getObservationVariableName(), "1"); + String originalValue = "1"; // Convenience variable, this value is reused. + newExp.put(traits.get(0).getObservationVariableName(), originalValue); + newExp.put(traits.get(1).getObservationVariableName(), "2"); importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); @@ -1140,10 +1216,15 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); - newObservation.put(traits.get(0).getObservationVariableName(), ""); - newObservation.put(traits.get(1).getObservationVariableName(), "2"); + newObservation.put(traits.get(0).getObservationVariableName(), ""); // This blank value should not overwrite. + newObservation.put(traits.get(1).getObservationVariableName(), "3"); // This valid value should overwrite. + newObservation.put(traits.get(2).getObservationVariableName(), "4"); // This valid new observation should be appended. - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + Map requestBody = new HashMap<>(); + requestBody.put("overwrite", "true"); + requestBody.put("overwriteReason", "testing"); + + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), requestBody, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -1154,6 +1235,9 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); + // The blank value should not have produced an overwrite. + // This change is to make the "expected" value correct. + newObservation.put(traits.get(0).getObservationVariableName(), originalValue); if(commit) { assertRowSaved(newObservation, program, traits); } else {