diff --git a/.env.template b/.env.template index 283b97128..0f100a7db 100644 --- a/.env.template +++ b/.env.template @@ -54,4 +54,7 @@ AWS_REGION= AWS_ACCESS_KEY_ID= AWS_SECRET_KEY= AWS_GENO_BUCKET= -AWS_S3_ENDPOINT= \ No newline at end of file +AWS_S3_ENDPOINT= + +BRAPI_VENDOR_SUBMISSION_ENABLED=false #can a submission be sent to a vendor via BrAPI +BRAPI_VENDOR_CHECK_FREQUENCY=1d #how often to check for vendor updates for sample submissions \ No newline at end of file diff --git a/localstack/startup_info.json b/localstack/startup_info.json new file mode 100755 index 000000000..a38d9d96e --- /dev/null +++ b/localstack/startup_info.json @@ -0,0 +1 @@ +[{"timestamp": "2022-11-17T17:01:36.401503", "localstack_version": "1.2.0", "localstack_ext_version": "1.2.0", "pro_activated": false}] \ No newline at end of file diff --git a/pom.xml b/pom.xml index b12c7b648..acf4e3071 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ 1.0-SNAPSHOT jar - Breeding Insight API + DeltaBreed API https://breedinginsight.org @@ -57,6 +57,10 @@ David Phillips drp227@cornell.edu + + Matthew Mandych + mlm483@cornell.edu + @@ -149,6 +153,13 @@ false + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots/ + + false + + diff --git a/settings.xml b/settings.xml index 4905dc995..c3867921d 100644 --- a/settings.xml +++ b/settings.xml @@ -49,13 +49,6 @@ true true - - github-brapi - GitHub Breeding Insight Apache Maven Packages - https://maven.pkg.github.com/Breeding-Insight/brapi - true - true - github-fannypack FannyPack github repository @@ -73,11 +66,6 @@ ${GITHUB_ACTOR} ${GITHUB_TOKEN} - - github-brapi - ${GITHUB_ACTOR} - ${GITHUB_TOKEN} - github-fannypack ${GITHUB_ACTOR} diff --git a/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java b/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java new file mode 100644 index 000000000..43923c71a --- /dev/null +++ b/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java @@ -0,0 +1,284 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.v1.controller.geno; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.*; +import io.micronaut.http.annotation.*; +import io.micronaut.http.server.types.files.StreamedFile; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.geno.BrAPIVendorOrderSubmission; +import org.breedinginsight.api.auth.*; +import org.breedinginsight.api.model.v1.response.DataResponse; +import org.breedinginsight.api.model.v1.response.Response; +import org.breedinginsight.api.model.v1.response.metadata.Metadata; +import org.breedinginsight.api.model.v1.response.metadata.Pagination; +import org.breedinginsight.api.model.v1.response.metadata.Status; +import org.breedinginsight.api.model.v1.response.metadata.StatusCode; +import org.breedinginsight.model.*; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.services.SampleSubmissionService; +import org.breedinginsight.services.UserService; +import org.breedinginsight.utilities.Utilities; + +import javax.annotation.Nullable; +import javax.inject.Inject; +import javax.validation.constraints.NotBlank; +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Slf4j +@Controller("/${micronaut.bi.api.version}") +@Secured(SecurityRule.IS_AUTHENTICATED) +public class SampleSubmissionController { + + private final boolean brapiSubmissionEnabled; + private final SampleSubmissionService sampleSubmissionService; + private final ProgramService programService; + private final SecurityService securityService; + private final UserService userService; + + private final Gson gson; + + @Inject + public SampleSubmissionController(@Property(name = "brapi.vendor-submission-enabled") boolean brapiSubmissionEnabled, SampleSubmissionService sampleSubmissionService, ProgramService programService, SecurityService securityService, UserService userService) { + this.brapiSubmissionEnabled = brapiSubmissionEnabled; + this.sampleSubmissionService = sampleSubmissionService; + this.programService = programService; + this.securityService = securityService; + this.userService = userService; + this.gson = new Gson(); + } + + @Get("programs/{programId}/submissions") + @Produces(MediaType.APPLICATION_JSON) + @ProgramSecured(roleGroups = ProgramSecuredRoleGroup.ALL) + public HttpResponse>> getProgramSampleSubmissions(@PathVariable UUID programId) { + Optional program = programService.getById(programId); + if(program.isEmpty()) { + log.info(String.format("programId not found: %s", programId.toString())); + return HttpResponse.notFound(); + } + + List submissions = sampleSubmissionService.getProgramSubmissions(program.get()); + Metadata metadata = new Metadata(new Pagination(submissions.size(), submissions.size(), 1, 0), + List.of(new Status(StatusCode.INFO, "Successful Query"))); + Response> response = new Response<>(metadata, new DataResponse<>(submissions)); + return HttpResponse.ok(response); + } + + @Get("programs/{programId}/submissions/{submissionId}") + @Produces(MediaType.APPLICATION_JSON) + @ProgramSecured(roleGroups = ProgramSecuredRoleGroup.ALL) + public HttpResponse> getSubmissionById(@PathVariable UUID programId, @PathVariable UUID submissionId, @QueryValue(value = "details", defaultValue = "false") @Nullable Boolean fetchDetails) { + Optional program = programService.getById(programId); + if(program.isEmpty()) { + log.info(String.format("programId not found: %s", programId.toString())); + return HttpResponse.notFound(); + } + + try { + Optional submission = sampleSubmissionService.getSampleSubmission(program.get(), submissionId, fetchDetails); + + if(submission.isEmpty()) { + return HttpResponse.notFound(); + } + + Metadata metadata = new Metadata(new Pagination(1, 1, 1, 0), + List.of(new Status(StatusCode.INFO, "Successful Query"))); + Response response = new Response<>(metadata, submission.get()); + return HttpResponse.ok(response); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); + } + } + + @Put("programs/{programId}/submissions/{submissionId}/status") + @Produces(MediaType.APPLICATION_JSON) + @ProgramSecured(roles = {ProgramSecuredRole.SYSTEM_ADMIN}) + public HttpResponse> updateSubmissionStatus(@PathVariable UUID programId, @PathVariable UUID submissionId, @Body String body) { + Optional program = programService.getById(programId); + if(program.isEmpty()) { + log.info(String.format("programId not found: %s", programId.toString())); + return HttpResponse.notFound(); + } + + AuthenticatedUser actingUser = securityService.getUser(); + Optional user = userService.getById(actingUser.getId()); + if (user.isEmpty()) { + return HttpResponse.unauthorized(); + } + + SampleSubmission.Status status = SampleSubmission.Status.fromValue(gson.fromJson(body, JsonObject.class).get("status").getAsString()); + if(status == null) { + HttpResponse response = HttpResponse.badRequest("Unrecognized status"); + return response; + } + + try { + Optional submission = sampleSubmissionService.updateSubmissionStatus(program.get(), submissionId, status, user.get()); + + if(submission.isEmpty()) { + return HttpResponse.notFound(); + } + + Metadata metadata = new Metadata(new Pagination(1, 1, 1, 0), + List.of(new Status(StatusCode.INFO, "Successful Update"))); + Response response = new Response<>(metadata, submission.get()); + return HttpResponse.ok(response); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); + } + } + + + @Get("/programs/{programId}/submissions/{submissionId}/dart") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + @Produces(value={"text/csv", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/octet-stream"}) + public HttpResponse generateDArTFile(@PathVariable UUID programId, @PathVariable UUID submissionId) { + try { + Optional program = programService.getById(programId); + if(program.isEmpty()) { + return HttpResponse.notFound(); + } + Optional downloadFile = sampleSubmissionService.generateDArTFile(program.get(), submissionId); + if(downloadFile.isEmpty()) { + return HttpResponse.notFound(); + } + HttpResponse response = HttpResponse + .ok(downloadFile.get().getStreamedFile()) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + downloadFile.get().getFileName()); + return response; + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); + } catch (IOException e) { + log.error("Error generating DArT file", e); + HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error generating DArT file").contentType(MediaType.TEXT_PLAIN).body("Error generating DArT file"); + return response; + } + } + + @Get("/programs/{programId}/submissions/{submissionId}/lookup") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + @Produces(value={"text/csv", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/octet-stream"}) + public HttpResponse generateLookupFile(@PathVariable UUID programId, @PathVariable UUID submissionId) { + try { + Optional program = programService.getById(programId); + if(program.isEmpty()) { + return HttpResponse.notFound(); + } + Optional downloadFile = sampleSubmissionService.generateLookupFile(program.get(), submissionId); + if(downloadFile.isEmpty()) { + return HttpResponse.notFound(); + } + HttpResponse response = HttpResponse + .ok(downloadFile.get().getStreamedFile()) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + downloadFile.get().getFileName()); + return response; + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); + } catch (IOException e) { + log.error("Error generating lookup file", e); + HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error generating lookup file").contentType(MediaType.TEXT_PLAIN).body("Error generating lookup file"); + return response; + } + } + + @Post("programs/{programId}/submissions/{submissionId}/submit") + @Produces(MediaType.APPLICATION_JSON) + @ProgramSecured(roles = {ProgramSecuredRole.SYSTEM_ADMIN}) + public HttpResponse> submitOrder(@PathVariable UUID programId, @PathVariable UUID submissionId, @QueryValue(value = "vendor") @NotBlank String vendorName) { + if(!brapiSubmissionEnabled) { + return HttpResponse.notFound(); + } + + try { + GenotypeVendor vendor = GenotypeVendor.fromName(vendorName); + if(vendor != null) { + Optional program = programService.getById(programId); + if (program.isEmpty()) { + return HttpResponse.notFound(); + } + + AuthenticatedUser actingUser = securityService.getUser(); + Optional user = userService.getById(actingUser.getId()); + if (user.isEmpty()) { + return HttpResponse.unauthorized(); + } + + Optional order = sampleSubmissionService.submitOrder(program.get(), submissionId, user.get(), vendor); + if (order.isEmpty()) { + return HttpResponse.notFound(); + } + + Metadata metadata = new Metadata(new Pagination(1, 1, 1, 0), + List.of(new Status(StatusCode.INFO, "Successful submission"))); + Response response = new Response<>(metadata, order.get()); + return HttpResponse.ok(response); + } else { + HttpResponse response = HttpResponse.badRequest("Unrecognized vendor"); + return response; + } + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); + } + } + + @Get("programs/{programId}/submissions/{submissionId}/status") + @Produces(MediaType.APPLICATION_JSON) + @ProgramSecured(roles = {ProgramSecuredRole.SYSTEM_ADMIN}) + public HttpResponse> checkVendorStatus(@PathVariable UUID programId, @PathVariable UUID submissionId) { + if(!brapiSubmissionEnabled) { + return HttpResponse.notFound(); + } + + Optional program = programService.getById(programId); + if(program.isEmpty()) { + log.info(String.format("programId not found: %s", programId.toString())); + return HttpResponse.notFound(); + } + + try { + Optional submission = sampleSubmissionService.checkVendorStatus(program.get(), submissionId); + + if(submission.isEmpty()) { + return HttpResponse.notFound(); + } + + Metadata metadata = new Metadata(new Pagination(1, 1, 1, 0), + List.of(new Status(StatusCode.INFO, "Successful Query"))); + Response response = new Response<>(metadata, submission.get()); + return HttpResponse.ok(response); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index 443b81b4c..c819cc10c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -97,19 +97,21 @@ public HttpResponse datasetExport( @QueryValue @Valid ExperimentExportQuery queryParams) { String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; try { - Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program does not exist")); + Optional program = programService.getById(programId); + if(program.isEmpty()) { + return HttpResponse.notFound(); + } // if a list of environmentIds are sent, return multiple files (zipped), // else if a single environmentId is sent, return single file (CSV/Excel), // else (if no environmentIds are sent), return a single file (CSV/Excel) including all Environments. - DownloadFile downloadFile = experimentService.exportObservations(program, experimentId, queryParams); + DownloadFile downloadFile = experimentService.exportObservations(program.get(), experimentId, queryParams); - HttpResponse response = HttpResponse + return HttpResponse .ok(downloadFile.getStreamedFile()) .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + downloadFile.getFileName()); - return response; } catch (Exception e) { - log.info(e.getMessage(), e); + log.info(downloadErrorMessage, e); HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, downloadErrorMessage).contentType(MediaType.TEXT_PLAIN).body(downloadErrorMessage); return response; } diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index d9daab6d0..821c39a98 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -48,4 +48,10 @@ public final class BrAPIAdditionalInfoFields { public static final String TREATMENTS = "treatments"; public static final String GID = "gid"; public static final String ENV_YEAR = "envYear"; + public static final String GERMPLASM_UUID = "germplasmId"; + public static final String SAMPLE_ORGANISM = "organism"; + public static final String SAMPLE_SPECIES = "species"; + public static final String OBS_UNIT_ID = "obsUnitID"; + public static final String GERMPLASM_NAME = "germplasmName"; + public static final String SUBMISSION_NAME = "submissionName"; } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index b902132ad..95d34cd32 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -29,9 +29,8 @@ import org.breedinginsight.model.*; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; -import org.breedinginsight.services.writers.CSVWriter; -import org.breedinginsight.services.writers.ExcelWriter; import org.breedinginsight.utilities.IntOrderComparator; +import org.breedinginsight.utilities.FileUtil; import org.breedinginsight.utilities.Utilities; import org.jetbrains.annotations.NotNull; @@ -242,7 +241,7 @@ public DownloadFile exportObservations( for (Map.Entry>> entry: rowsByStudyId.entrySet()) { List> rows = entry.getValue(); sortDefaultForExportRows(rows); - StreamedFile streamedFile = writeToStreamedFile(columns, rows, fileType, SHEET_NAME); + StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, rows, fileType, SHEET_NAME); String name = makeFileName(experiment, program, studyByDbId.get(entry.getKey()).getStudyName()) + fileType.getExtension(); // Add to file list. files.add(new DownloadFile(name, streamedFile)); @@ -261,7 +260,7 @@ public DownloadFile exportObservations( List> exportRows = new ArrayList<>(rowByOUId.values()); sortDefaultForExportRows(exportRows); // write export data to requested file format - StreamedFile streamedFile = writeToStreamedFile(columns, exportRows, fileType, SHEET_NAME); + StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, exportRows, fileType, SHEET_NAME); // Set filename. String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); String fileName = makeFileName(experiment, program, envFilenameFragment) + fileType.getExtension(); @@ -271,14 +270,6 @@ public DownloadFile exportObservations( return downloadFile; } - private StreamedFile writeToStreamedFile(List columns, List> data, FileType extension, String sheetName) throws IOException { - if (extension.equals(FileType.CSV)){ - return CSVWriter.writeToDownload(columns, data, extension); - } else { - return ExcelWriter.writeToDownload(sheetName, columns, data, extension); - } - } - private StreamedFile zipFiles(List files) throws IOException { PipedInputStream in = new PipedInputStream(); final PipedOutputStream out = new PipedOutputStream(in); diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIPlateDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIPlateDAO.java new file mode 100644 index 000000000..d4bd47490 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIPlateDAO.java @@ -0,0 +1,98 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.daos; + +import io.micronaut.context.annotation.Property; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.modules.genotype.PlatesApi; +import org.brapi.v2.model.geno.BrAPIPlate; +import org.brapi.v2.model.geno.request.BrAPIPlateNewRequest; +import org.brapi.v2.model.geno.request.BrAPIPlateSearchRequest; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.List; +import java.util.stream.Collectors; + +@Slf4j +@Singleton +public class BrAPIPlateDAO { + + private final String referenceSource; + + private final ProgramDAO programDAO; + private final ImportDAO importDAO; + private final BrAPIDAOUtil brAPIDAOUtil; + private final BrAPIEndpointProvider brAPIEndpointProvider; + + @Inject + public BrAPIPlateDAO(ProgramDAO programDAO, + ImportDAO importDAO, + BrAPIDAOUtil brAPIDAOUtil, + BrAPIEndpointProvider brAPIEndpointProvider, + @Property(name = "brapi.server.reference-source") String referenceSource) { + this.referenceSource = referenceSource; + this.programDAO = programDAO; + this.importDAO = importDAO; + this.brAPIDAOUtil = brAPIDAOUtil; + this.brAPIEndpointProvider = brAPIEndpointProvider; + } + + public List createPlates(Program program, List platesToSave, ImportUpload upload) throws ApiException { + PlatesApi platesApi = brAPIEndpointProvider.get(programDAO.getSampleClient(program.getId()), PlatesApi.class); + List newPlatesRequests = platesToSave.stream() + .map(plate -> new BrAPIPlateNewRequest().additionalInfo(plate.getAdditionalInfo()) + .externalReferences(plate.getExternalReferences()) + .plateBarcode(plate.getPlateBarcode()) + .plateFormat(plate.getPlateFormat()) + .plateName(plate.getPlateName()) + .programDbId(plate.getProgramDbId()) + .sampleType(plate.getSampleType()) + .studyDbId(plate.getStudyDbId()) + .trialDbId(plate.getTrialDbId()) + ) + .collect(Collectors.toList()); + return brAPIDAOUtil.post(newPlatesRequests, upload, platesApi::platesPost, importDAO::update); + } + + public List readPlatesByIds(Program program, List plateExternalIds) throws ApiException { + PlatesApi platesApi = brAPIEndpointProvider.get(programDAO.getSampleClient(program.getId()), PlatesApi.class); + + BrAPIPlateSearchRequest request = new BrAPIPlateSearchRequest().externalReferenceIDs(plateExternalIds) + .externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATES))); + + return brAPIDAOUtil.search(platesApi::searchPlatesPost, platesApi::searchPlatesSearchResultsDbIdGet, request); + } + + public List readPlatesBySubmissionIds(Program program, List submissionExternalIds) throws ApiException { + PlatesApi platesApi = brAPIEndpointProvider.get(programDAO.getSampleClient(program.getId()), PlatesApi.class); + + BrAPIPlateSearchRequest request = new BrAPIPlateSearchRequest().externalReferenceIDs(submissionExternalIds) + .externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATE_SUBMISSIONS))); + + return brAPIDAOUtil.search(platesApi::searchPlatesPost, platesApi::searchPlatesSearchResultsDbIdGet, request); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISampleDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISampleDAO.java new file mode 100644 index 000000000..1cb9a531a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISampleDAO.java @@ -0,0 +1,105 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.daos; + +import io.micronaut.context.annotation.Property; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.modules.genotype.SamplesApi; +import org.brapi.v2.model.geno.BrAPISample; +import org.brapi.v2.model.geno.request.BrAPISampleSearchRequest; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +@Slf4j +@Singleton +public class BrAPISampleDAO { + + private final String referenceSource; + + private final ProgramDAO programDAO; + private final ImportDAO importDAO; + private final BrAPIDAOUtil brAPIDAOUtil; + private final BrAPIEndpointProvider brAPIEndpointProvider; + + @Inject + public BrAPISampleDAO(ProgramDAO programDAO, + ImportDAO importDAO, + BrAPIDAOUtil brAPIDAOUtil, + BrAPIEndpointProvider brAPIEndpointProvider, + @Property(name = "brapi.server.reference-source") String referenceSource) { + this.referenceSource = referenceSource; + this.programDAO = programDAO; + this.importDAO = importDAO; + this.brAPIDAOUtil = brAPIDAOUtil; + this.brAPIEndpointProvider = brAPIEndpointProvider; + } + + public List createSamples(Program program, List samplesToSave, ImportUpload upload) throws ApiException { + SamplesApi samplesApi = brAPIEndpointProvider.get(programDAO.getSampleClient(program.getId()), SamplesApi.class); + + return brAPIDAOUtil.post(samplesToSave, upload, samplesApi::samplesPost, importDAO::update); + } + + public List readSamplesByIds(Program program, List sampleExternalIds) throws ApiException { + if(sampleExternalIds.isEmpty()) { + return Collections.emptyList(); + } + + BrAPISampleSearchRequest searchRequest = new BrAPISampleSearchRequest().externalReferenceIDs(sampleExternalIds) + .externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.SAMPLES))); + + SamplesApi samplesApi = brAPIEndpointProvider.get(programDAO.getSampleClient(program.getId()), SamplesApi.class); + return brAPIDAOUtil.search(samplesApi::searchSamplesPost, samplesApi::searchSamplesSearchResultsDbIdGet, searchRequest); + } + + public List readSamplesByPlateIds(Program program, List plateExternalIds) throws ApiException { + if(plateExternalIds.isEmpty()) { + return Collections.emptyList(); + } + + BrAPISampleSearchRequest searchRequest = new BrAPISampleSearchRequest().externalReferenceIDs(plateExternalIds) + .externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATES))); + + SamplesApi samplesApi = brAPIEndpointProvider.get(programDAO.getSampleClient(program.getId()), SamplesApi.class); + return brAPIDAOUtil.search(samplesApi::searchSamplesPost, samplesApi::searchSamplesSearchResultsDbIdGet, searchRequest); + } + + public List readSamplesBySubmissionIds(Program program, List submissionExternalIds) throws ApiException { + if(submissionExternalIds.isEmpty()) { + return Collections.emptyList(); + } + + BrAPISampleSearchRequest searchRequest = new BrAPISampleSearchRequest().externalReferenceIDs(submissionExternalIds) + .externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATE_SUBMISSIONS))); + + SamplesApi samplesApi = brAPIEndpointProvider.get(programDAO.getSampleClient(program.getId()), SamplesApi.class); + return brAPIDAOUtil.search(samplesApi::searchSamplesPost, samplesApi::searchSamplesSearchResultsDbIdGet, searchRequest); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index eb4832b67..2bf412955 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -30,24 +30,24 @@ import java.util.List; -public abstract class BrAPIImportService { - public String getImportTypeId() {return null;} - public BrAPIImport getImportClass() {return null;} - public String getInvalidIntegerMsg(String columnName) { +public interface BrAPIImportService { + String getImportTypeId(); + BrAPIImport getImportClass(); + default String getInvalidIntegerMsg(String columnName) { return String.format("Column name \"%s\" must be integer type, but non-integer type provided.", columnName); } - public String getBlankRequiredFieldMsg(String fieldName) { + default String getBlankRequiredFieldMsg(String fieldName) { return String.format("Required field \"%s\" cannot contain empty values", fieldName); } - public String getMissingColumnMsg(String columnName) { + default String getMissingColumnMsg(String columnName) { return String.format("Column name \"%s\" does not exist in file", columnName); } - public String getMissingUserInputMsg(String fieldName) { + default String getMissingUserInputMsg(String fieldName) { return String.format("User input, \"%s\" is required", fieldName); } - public String getWrongUserInputDataTypeMsg(String fieldName, String typeName) { + default String getWrongUserInputDataTypeMsg(String fieldName, String typeName) { return String.format("User input, \"%s\" must be an %s", fieldName, typeName); } - public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) - throws UnprocessableEntityException, DoesNotExistException, ValidatorException, ApiException, MissingRequiredInfoException {return null;} + ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + throws UnprocessableEntityException, DoesNotExistException, ValidatorException, ApiException, MissingRequiredInfoException; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java index f3d4f3180..e478fc385 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java @@ -21,9 +21,10 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; -import org.brapi.v2.model.core.BrAPILocation; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.geno.BrAPIPlate; +import org.brapi.v2.model.geno.BrAPISample; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; @@ -44,6 +45,8 @@ public class PendingImport { private PendingImportObject study; private PendingImportObject observationUnit; private List> observations = new ArrayList<>(); + private PendingImportObject plate; + private PendingImportObject sample; @JsonIgnore public PendingImportObject getObservation() { diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 4978ed9b4..aa425a2da 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -39,7 +39,7 @@ @Singleton @Slf4j -public class ExperimentImportService extends BrAPIImportService { +public class ExperimentImportService implements BrAPIImportService { private final String IMPORT_TYPE_ID = "ExperimentImport"; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index cadd379a0..e4084eae4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -39,7 +39,7 @@ @Singleton @Slf4j -public class GermplasmImportService extends BrAPIImportService { +public class GermplasmImportService implements BrAPIImportService { private final String IMPORT_TYPE_ID = "GermplasmImport"; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java new file mode 100644 index 000000000..6ae9fc2ff --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java @@ -0,0 +1,207 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.imports.sample; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.geno.BrAPIPlate; +import org.brapi.v2.model.geno.BrAPIPlateFormat; +import org.brapi.v2.model.geno.BrAPISample; +import org.brapi.v2.model.geno.BrAPISampleTypeEnum; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.config.*; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import org.breedinginsight.utilities.Utilities; + +import java.util.*; + +@Getter +@Setter +@NoArgsConstructor +@ImportConfigMetadata(id = "SampleImport", name = "Sample Import", + description = "This import is used to create Genotype Samples") +public class SampleSubmissionImport implements BrAPIImport { + + private static final String SAMPLE_NAME_FORMAT = "S_%s__%s_%s%02d"; + private static final String SAMPLE_NAME_ILLEGAL_CHARS_REGEX = "[^a-zA-Z0-9_]"; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "plateId", name = Columns.PLATE_ID, description = "The ID which uniquely identifies this plate to the client making the request") + private String plateId; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "row", name = Columns.ROW, description = "The Row identifier for this samples location in the plate, ex: B") + private String row; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "column", name = Columns.COLUMN, description = "The Column identifier for this samples location in the plate, ex: 6") + private String column; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "organism", name = Columns.ORGANISM, description = "Scientific organism name") + private String organism; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "species", name = Columns.SPECIES, description = "Scientific species name") + private String species; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "germplasmName", name = Columns.GERMPLASM_NAME, description = "Name of germplasm") + private String germplasmName; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "gid", name = Columns.GERMPLASM_GID, description = "Unique germplasm identifier") + private String gid; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "obsUnitId", name = Columns.OBS_UNIT_ID, description = "The Observation Unit that this sample was collected from") + private String obsUnitId; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "tissue", name = Columns.TISSUE, description = "The type of tissue in this sample. List of accepted tissue types can be found in the Vendor Specs.") + private String tissue; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "comments", name = Columns.COMMENTS, description = "Generic comments about this sample for the vendor") + private String comments; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT, collectTime = ImportCollectTimeEnum.UPLOAD) + @ImportMappingRequired + @ImportFieldMetadata(id="submissionName", name="Submission Name", description = "Name of the submission imported.") + private String submissionName; + + public static final class Columns { + public static final String PLATE_ID = "PlateID"; + public static final String ROW = "Row"; + public static final String COLUMN = "Column"; + public static final String ORGANISM = "Organism"; + public static final String SPECIES = "Species"; + public static final String GERMPLASM_NAME = "Germplasm Name"; + public static final String GERMPLASM_GID = "GID"; + public static final String TISSUE = "Tissue"; + public static final String COMMENTS = "Comments"; + public static final String OBS_UNIT_ID = "ObsUnitID"; + } + + public BrAPISample constructBrAPISample(boolean commit, Program program, User user, BrAPIPlate plate, String referenceSource, String submissionId, BrAPIGermplasm germplasm, BrAPIObservationUnit ou) { + List xrefs = new ArrayList<>(); + xrefs.add(new BrAPIExternalReference().referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS)) + .referenceId(program.getId() + .toString())); + String germXrefSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.GERMPLASM); + xrefs.add(germplasm.getExternalReferences() + .stream() + .filter(xref -> xref.getReferenceSource().equals(referenceSource) || xref.getReferenceSource().equals(germXrefSource)) + .findFirst() + .orElseThrow(() -> new IllegalStateException(String.format("Germplasm %s doesn't have an xref! -> %s", germplasm.getAccessionNumber(), germplasm.toString()))).referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.GERMPLASM))); + if(commit) { + xrefs.add(new BrAPIExternalReference().referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.SAMPLES)) + .referenceId(UUID.randomUUID().toString())); + xrefs.add(Utilities.getExternalReference(plate.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATES)) + .get()); + } + if (submissionId != null) { + xrefs.add(new BrAPIExternalReference().referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATE_SUBMISSIONS)) + .referenceId(submissionId)); + } + + Map createdBy = new HashMap<>(); + createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_ID, + user.getId().toString()); + createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_NAME, user.getName()); + + BrAPISample brAPISample = new BrAPISample() + .putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_BY, createdBy) + .putAdditionalInfoItem(BrAPIAdditionalInfoFields.SAMPLE_SPECIES, species) + .putAdditionalInfoItem(BrAPIAdditionalInfoFields.SAMPLE_ORGANISM, organism) + .putAdditionalInfoItem(BrAPIAdditionalInfoFields.GID, germplasm.getAccessionNumber()) + .putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_NAME, germplasm.getDefaultDisplayName()) + .externalReferences(xrefs) + .plateName(plateId) + .plateDbId(plate.getPlateDbId()) + .sampleName(generateSampleName(germplasm, plate)) + .germplasmDbId(germplasm.getGermplasmDbId()) + .row(row.toUpperCase()) + .column(Integer.valueOf(column)) + .tissueType(tissue) + .sampleDescription(comments); + + if (ou != null) { + brAPISample + .putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBS_UNIT_ID, + Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS)) + .get() + .getReferenceId()) + .observationUnitDbId(ou.getObservationUnitDbId()); + } + + return brAPISample; + } + + private String generateSampleName(BrAPIGermplasm germplasm, BrAPIPlate plate) { + var legalGermplasmName = germplasm.getDefaultDisplayName().replaceAll(SAMPLE_NAME_ILLEGAL_CHARS_REGEX, "_"); + String name = String.format(SAMPLE_NAME_FORMAT, legalGermplasmName, plate.getPlateName(), row, Integer.parseInt(column)); + + if(name.length() > 100) { + //TODO what to do in this case?? + throw new RuntimeException("Generated sample name is too long"); + } + + return name; + } + + public BrAPIPlate constructBrAPIPlate(boolean commit, Program program, User user, String referenceSource, String submissionId) { + List xrefs = new ArrayList<>(); + xrefs.add(new BrAPIExternalReference().referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS)) + .referenceId(program.getId() + .toString())); + + BrAPIPlate brAPIPlate = new BrAPIPlate(); + if(commit) { + xrefs.add(new BrAPIExternalReference().referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATES)) + .referenceId(UUID.randomUUID().toString())); + } + if (submissionId != null) { + xrefs.add(new BrAPIExternalReference().referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATE_SUBMISSIONS)) + .referenceId(submissionId)); + brAPIPlate.putAdditionalInfoItem(BrAPIAdditionalInfoFields.SUBMISSION_NAME, submissionName); + } + Map createdBy = new HashMap<>(); + createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_ID, + user.getId() + .toString()); + createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_NAME, user.getName()); + + + + return brAPIPlate + .externalReferences(xrefs) + .putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_BY, createdBy) + .plateName(plateId) + .sampleType(BrAPISampleTypeEnum.fromValue(tissue.toUpperCase())) + .plateFormat(BrAPIPlateFormat.PLATE_96); + + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java new file mode 100644 index 000000000..637fa7ce3 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java @@ -0,0 +1,76 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.imports.sample; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.services.processors.Processor; +import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; +import org.breedinginsight.brapps.importer.services.processors.SampleSubmissionProcessor; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.services.exceptions.ValidatorException; +import tech.tablesaw.api.Table; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.List; + +@Singleton +@Slf4j +public class SampleSubmissionImportService implements BrAPIImportService { + private final String IMPORT_TYPE_ID = "SampleImport"; + private final Provider sampleProcessorProvider; + private final Provider processorManagerProvider; + + @Inject + public SampleSubmissionImportService(Provider sampleProcessorProvider, Provider processorManagerProvider) + { + this.sampleProcessorProvider = sampleProcessorProvider; + this.processorManagerProvider = processorManagerProvider; + } + + @Override + public String getImportTypeId() { + return IMPORT_TYPE_ID; + } + + @Override + public BrAPIImport getImportClass() { + return new SampleSubmissionImport(); + } + + @Override + public ImportPreviewResponse process(List brAPIImports, + Table data, + Program program, + ImportUpload upload, + User user, + Boolean commit) throws UnprocessableEntityException, DoesNotExistException, ValidatorException, ApiException, MissingRequiredInfoException { + List processors = List.of(sampleProcessorProvider.get()); + return processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/ExternalReferenceSource.java b/src/main/java/org/breedinginsight/brapps/importer/services/ExternalReferenceSource.java index 78e051e08..45553e189 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/ExternalReferenceSource.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/ExternalReferenceSource.java @@ -10,7 +10,11 @@ public enum ExternalReferenceSource { OBSERVATION_UNITS("observationunits"), DATASET("dataset"), LISTS("lists"), - OBSERVATIONS("observations"); + OBSERVATIONS("observations"), + GERMPLASM("germplasm"), + PLATES("plates"), + SAMPLES("samples"), + PLATE_SUBMISSIONS("plates/submissions"); private String name; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 74051b813..05c48601d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -156,7 +156,8 @@ private Table parseUploadedFile(CompletedFileUpload file) throws UnsupportedType throw new HttpStatusException(HttpStatus.BAD_REQUEST, "Error parsing csv: " + e.getMessage()); } } else if (mediaType.toString().equals(SupportedMediaType.XLS) || - mediaType.toString().equals(SupportedMediaType.XLSX)) { + mediaType.toString().equals(SupportedMediaType.XLSX) || + mediaType.toString().equals(SupportedMediaType.XLSB)) { try { //TODO: Allow them to pass in header row index in the future @@ -453,7 +454,7 @@ private void processFile(List finalBrAPIImportList, Table data, Pro progress.setUpdatedBy(actingUser.getId()); importDAO.update(upload); }catch (ValidatorException e) { - log.info("Validation errors", e); + log.info("Validation errors: \n" + e); ImportProgress progress = upload.getProgress(); progress.setStatuscode((short) HttpStatus.UNPROCESSABLE_ENTITY.getCode()); progress.setMessage("Multiple Errors"); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java new file mode 100644 index 000000000..d92ebb300 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java @@ -0,0 +1,275 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors; + +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.geno.BrAPIPlate; +import org.brapi.v2.model.geno.BrAPISample; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.daos.BrAPIPlateDAO; +import org.breedinginsight.brapps.importer.daos.BrAPISampleDAO; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.dao.db.tables.daos.SampleSubmissionDao; +import org.breedinginsight.dao.db.tables.pojos.SampleSubmissionEntity; +import org.breedinginsight.daos.SampleSubmissionDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.SampleSubmission; +import org.breedinginsight.model.User; +import org.breedinginsight.services.SampleSubmissionService; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.utilities.Utilities; +import org.jooq.DSLContext; +import tech.tablesaw.api.Table; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +@Prototype +public class SampleSubmissionProcessor implements Processor { + + private static final String MISSING_REQUIRED_DATA = "Missing required data"; + private static final String MISSING_GERM_ASSOCIATION = "One of GID or ObsUnitID is required"; + private static final String UNKNOWN_OBS_UNIT_ID = "Unknown ObsUnitID"; + private static final String UNKNOWN_GID = "Unknown germplasm GID"; + private static final String INVALID_COLUMN = "Column must be a number between 1 and 12"; + private static final String INVALID_ROW = "Row must be a letter between A and H"; + private static final String MULTIPLE_SAMPLES_SINGLE_WELL = "The sample in row %d is already in row: %s, column: %d"; + private final String referenceSource; + private final BrAPIGermplasmDAO germplasmDAO; + private final BrAPIObservationUnitDAO observationUnitDAO; + private final SampleSubmissionService sampleSubmissionService; + private SampleSubmission submission; + private Map germplasmByGID = new HashMap<>(); + private Map germplasmByDbId = new HashMap<>(); + private Map observationUnitsById = new HashMap<>(); + private Map> plateById = new HashMap<>(); + private Map plateLayouts = new HashMap<>(); + + @Inject + public SampleSubmissionProcessor(@Property(name = "brapi.server.reference-source") String referenceSource, + BrAPIGermplasmDAO germplasmDAO, + BrAPIObservationUnitDAO observationUnitDAO, + SampleSubmissionService sampleSubmissionService) { + this.referenceSource = referenceSource; + this.germplasmDAO = germplasmDAO; + this.observationUnitDAO = observationUnitDAO; + this.sampleSubmissionService = sampleSubmissionService; + } + + @Override + public void getExistingBrapiData(List importRows, Program program) throws ValidatorException, ApiException { + Set gids = importRows.stream() + .filter((row -> StringUtils.isNotBlank(((SampleSubmissionImport) row).getGid()))) + .map(row -> ((SampleSubmissionImport) row).getGid()) + .collect(Collectors.toSet()); + + List germplasm = germplasmDAO.getGermplasm(program.getId()); + + List obsUnitIds = importRows.stream() + .filter((row -> StringUtils.isNotBlank(((SampleSubmissionImport) row).getObsUnitId()))) + .map(row -> ((SampleSubmissionImport) row).getObsUnitId()) + .distinct() + .collect(Collectors.toList()); + + List observationUnits = observationUnitDAO.getObservationUnitsById(obsUnitIds, program); + Set germDbIds = new HashSet<>(); + String ouRefSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS); + observationUnits.forEach(ou -> { + observationUnitsById.put(Utilities.getExternalReference(ou.getExternalReferences(), ouRefSource) + .get() + .getReferenceId(), ou); + germDbIds.add(ou.getGermplasmDbId()); + }); + germplasm.stream() + .filter(germ -> gids.contains(germ.getAccessionNumber()) || germDbIds.contains(germ.getGermplasmDbId())) + .forEach(germ -> { + germplasmByGID.put(germ.getAccessionNumber(), germ); + germplasmByDbId.put(germ.getGermplasmDbId(), germ); + }); + } + + @Override + public Map process(ImportUpload upload, + List importRows, + Map mappedBrAPIImport, + Table data, + Program program, + User user, + boolean commit) throws ValidatorException, MissingRequiredInfoException, ApiException { + ValidationErrors validationErrors = new ValidationErrors(); + + String submissionId = null; + if (commit) { + submissionId = UUID.randomUUID() + .toString(); + SampleSubmissionImport row = (SampleSubmissionImport) importRows.get(0); + submission = SampleSubmission.builder() + .id(UUID.fromString(submissionId)) + .name(row.getSubmissionName()) + .createdBy(user.getId()) + .build(); + } + + for (int i = 0; i < importRows.size(); i++) { + int rowNum = i + 2; //accounts for column header and 0-index of i + SampleSubmissionImport row = (SampleSubmissionImport) importRows.get(i); + + if (validRow(row, rowNum, validationErrors)) { + PendingImport pendingImport = new PendingImport(); + PendingImportObject plate = plateById.getOrDefault(row.getPlateId(), new PendingImportObject<>(ImportObjectState.NEW, row.constructBrAPIPlate(commit, program, user, referenceSource, submissionId))); + pendingImport.setPlate(plate); + plateById.putIfAbsent(plate.getBrAPIObject() + .getPlateName(), plate); + + BrAPIGermplasm germ; + if (StringUtils.isNotBlank(row.getObsUnitId())) { + germ = germplasmByDbId.get(observationUnitsById.get(row.getObsUnitId()) + .getGermplasmDbId()); + } else { + germ = germplasmByGID.get(row.getGid()); + } + pendingImport.setSample(new PendingImportObject<>(ImportObjectState.NEW, + row.constructBrAPISample(commit, program, user, plate.getBrAPIObject(), referenceSource, submissionId, germ, observationUnitsById.get(row.getObsUnitId())))); + mappedBrAPIImport.put(rowNum, pendingImport); + } + } + + if (validationErrors.hasErrors()) { + throw new ValidatorException(validationErrors); + } + + return Map.of("Plates", + ImportPreviewStatistics.builder() + .newObjectCount(plateById.size()) + .build(), + "Samples", + ImportPreviewStatistics.builder() + .newObjectCount(importRows.size()) + .build()); + } + + private boolean validRow(SampleSubmissionImport row, int rowNum, ValidationErrors validationErrors) { + int numErrorsBefore = validationErrors.getRowErrors().size(); + if (StringUtils.isBlank(row.getPlateId())) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.PLATE_ID, MISSING_REQUIRED_DATA, HttpStatus.UNPROCESSABLE_ENTITY)); + } + int plateRow = -1; + if (StringUtils.isBlank(row.getRow())) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.ROW, MISSING_REQUIRED_DATA, HttpStatus.UNPROCESSABLE_ENTITY)); + } else if (row.getRow().length() > 1 + || row.getRow().toUpperCase().charAt(0) - 'A' < 0 + || row.getRow().toUpperCase().charAt(0) - 'H' > 0) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.ROW, INVALID_ROW, HttpStatus.UNPROCESSABLE_ENTITY)); + } else { + plateRow = row.getRow().toUpperCase().charAt(0) - 'A'; + } + int plateCol = -1; + if (StringUtils.isBlank(row.getColumn())) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.COLUMN, MISSING_REQUIRED_DATA, HttpStatus.UNPROCESSABLE_ENTITY)); + } else { + try { + plateCol = Integer.parseInt(row.getColumn()); + if (plateCol < 1 || plateCol > 12) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.COLUMN, INVALID_COLUMN, HttpStatus.UNPROCESSABLE_ENTITY)); + plateCol = -1; + } + } catch (NumberFormatException e) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.COLUMN, INVALID_COLUMN, HttpStatus.UNPROCESSABLE_ENTITY)); + } + } + if (StringUtils.isBlank(row.getOrganism())) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.ORGANISM, MISSING_REQUIRED_DATA, HttpStatus.UNPROCESSABLE_ENTITY)); + } + if (StringUtils.isBlank(row.getGid()) && StringUtils.isBlank(row.getObsUnitId())) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.GERMPLASM_GID, MISSING_GERM_ASSOCIATION, HttpStatus.UNPROCESSABLE_ENTITY)); + } else if (StringUtils.isNotBlank(row.getObsUnitId()) && !observationUnitsById.containsKey(row.getObsUnitId())) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.OBS_UNIT_ID, UNKNOWN_OBS_UNIT_ID, HttpStatus.UNPROCESSABLE_ENTITY)); + } else if (StringUtils.isNotBlank(row.getGid()) && !germplasmByGID.containsKey(row.getGid())) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.GERMPLASM_GID, UNKNOWN_GID, HttpStatus.UNPROCESSABLE_ENTITY)); + } + if (StringUtils.isBlank(row.getTissue())) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.TISSUE, MISSING_REQUIRED_DATA, HttpStatus.UNPROCESSABLE_ENTITY)); + } + + if (plateRow > -1 && plateCol > -1) { + int[][] plateLayout = plateLayouts.getOrDefault(row.getPlateId(), new int[9][13]); + if (plateLayout[plateRow][plateCol] > 0) { + validationErrors.addError(rowNum, + new ValidationError(SampleSubmissionImport.Columns.ROW + "/" + SampleSubmissionImport.Columns.COLUMN, + String.format(MULTIPLE_SAMPLES_SINGLE_WELL, plateLayout[plateRow][plateCol], Character.toString('A' + plateRow), plateCol), + HttpStatus.UNPROCESSABLE_ENTITY)); + } else { + plateLayout[plateRow][plateCol] = rowNum; + plateLayouts.put(row.getPlateId(), plateLayout); + } + } + + return numErrorsBefore == validationErrors.getRowErrors() + .size(); + } + + @Override + public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { + + } + + @Override + public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) throws ValidatorException { + List platesToSave = plateById.values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toList()); + List samplesToSave = mappedBrAPIImport.values().stream().map(row -> row.getSample().getBrAPIObject()).collect(Collectors.toList()); + + submission.setPlates(platesToSave); + submission.setSamples(samplesToSave); + + try { + sampleSubmissionService.createSubmission(submission, program, upload); + } catch (ApiException e) { + log.error("Error saving sample submission import: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving sample submission import", e); + } catch (Exception e) { + log.error("Error saving sample submission import", e); + throw new InternalServerException(e.getMessage(), e); + } + } + + @Override + public String getName() { + return "SampleSubmission"; + } +} diff --git a/src/main/java/org/breedinginsight/daos/ProgramDAO.java b/src/main/java/org/breedinginsight/daos/ProgramDAO.java index 3d3600da7..e60481f35 100644 --- a/src/main/java/org/breedinginsight/daos/ProgramDAO.java +++ b/src/main/java/org/breedinginsight/daos/ProgramDAO.java @@ -62,4 +62,6 @@ public interface ProgramDAO extends DAO { BrAPIClient getCoreClient(UUID programId); BrAPIClient getPhenoClient(UUID programId); + + BrAPIClient getSampleClient(UUID programId); } diff --git a/src/main/java/org/breedinginsight/daos/SampleSubmissionDAO.java b/src/main/java/org/breedinginsight/daos/SampleSubmissionDAO.java new file mode 100644 index 000000000..a6f84596c --- /dev/null +++ b/src/main/java/org/breedinginsight/daos/SampleSubmissionDAO.java @@ -0,0 +1,123 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.daos; + +import org.brapi.v2.model.geno.response.BrAPIVendorOrderStatusResponseResult; +import org.breedinginsight.dao.db.Tables; +import org.breedinginsight.dao.db.tables.BiUserTable; +import org.breedinginsight.dao.db.tables.daos.SampleSubmissionDao; +import org.breedinginsight.dao.db.tables.pojos.SampleSubmissionEntity; +import org.breedinginsight.dao.db.tables.records.SampleSubmissionRecord; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.SampleSubmission; +import org.breedinginsight.model.User; +import org.jooq.*; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.time.OffsetDateTime; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.breedinginsight.dao.db.Tables.*; +import static org.breedinginsight.dao.db.Tables.BI_USER; + +@Singleton +public class SampleSubmissionDAO extends SampleSubmissionDao { + + private DSLContext dsl; + + @Inject + public SampleSubmissionDAO(Configuration config, DSLContext dsl) { + super(config); + this.dsl = dsl; + } + + public List getBySubmissionId(Program program, UUID submissionId) { + return getRecords(List.of(SAMPLE_SUBMISSION.PROGRAM_ID.eq(program.getId()), SAMPLE_SUBMISSION.ID.eq(submissionId))); + } + + public List getByProgramId(UUID programId) { + return getRecords(List.of(SAMPLE_SUBMISSION.PROGRAM_ID.eq(programId))); + } + + public List getSubmittedAndNotComplete() { + return getRecords(List.of(SAMPLE_SUBMISSION.VENDOR_ORDER_ID.isNotNull(), + SAMPLE_SUBMISSION.VENDOR_STATUS.isNull().or(SAMPLE_SUBMISSION.VENDOR_STATUS.ne(BrAPIVendorOrderStatusResponseResult.StatusEnum.COMPLETED.name())))); + } + + public SampleSubmission update(SampleSubmission submission, User updatedBy) { + submission.setUpdatedAt(OffsetDateTime.now()); + submission.setUpdatedBy(updatedBy.getId()); + if(submission.getSubmittedByUser() != null) { + submission.setSubmittedBy(submission.getSubmittedByUser().getId()); + } else { + submission.setSubmittedBy(null); + } + + super.update(submission); + return submission; + } + + private List getRecords(List andConditions) { + BiUserTable createdByUser = BI_USER.as("createdByUser"); + BiUserTable updatedByUser = BI_USER.as("updatedByUser"); + BiUserTable submittedByUser = BI_USER.as("submittedByUser"); + try(SelectSelectStep select = dsl.select()) { + SelectConditionStep query = select + .from(SAMPLE_SUBMISSION) + .join(createdByUser).on(SAMPLE_SUBMISSION.CREATED_BY.eq(createdByUser.ID)) + .join(updatedByUser).on(SAMPLE_SUBMISSION.UPDATED_BY.eq(updatedByUser.ID)) + .leftJoin(submittedByUser).on(SAMPLE_SUBMISSION.SUBMITTED_BY.eq(submittedByUser.ID)) + .where("1=1"); + + for (Condition condition : andConditions) { + query = query.and(condition); + } + + return query.fetch() + .stream() + .map(record -> parseRecord(record, createdByUser, updatedByUser, submittedByUser)) + .collect(Collectors.toList()); + } + + } + + + private SampleSubmission parseRecord(Record record, BiUserTable createdByUser, BiUserTable updatedByUser, BiUserTable submittedByUser) { + SampleSubmission submission = new SampleSubmission(record.into(SampleSubmissionEntity.class)); + submission.setCreatedByUser(User.parseSQLRecord(record, createdByUser)); + submission.setUpdatedByUser(User.parseSQLRecord(record, updatedByUser)); + submission.setSubmittedByUser(User.parseSQLRecord(record, submittedByUser)); + //these explicit setters are needed because of overlapping column names with the bi_user table joined to the query + submission.setId(record.get(SAMPLE_SUBMISSION.ID)); + submission.setName(record.get(SAMPLE_SUBMISSION.NAME)); + submission.setProgramId(record.get(SAMPLE_SUBMISSION.PROGRAM_ID)); + submission.setCreatedAt(record.get(SAMPLE_SUBMISSION.CREATED_AT)); + submission.setUpdatedAt(record.get(SAMPLE_SUBMISSION.UPDATED_AT)); + submission.setCreatedBy(submission.getCreatedByUser().getId()); + submission.setUpdatedBy(submission.getUpdatedByUser().getId()); + if(submission.getSubmittedByUser() != null) { + submission.setSubmittedBy(submission.getSubmittedByUser().getId()); + } + return submission; + } +} diff --git a/src/main/java/org/breedinginsight/daos/impl/ProgramDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/ProgramDAOImpl.java index 5818ab281..21231a05d 100644 --- a/src/main/java/org/breedinginsight/daos/impl/ProgramDAOImpl.java +++ b/src/main/java/org/breedinginsight/daos/impl/ProgramDAOImpl.java @@ -36,7 +36,6 @@ import org.breedinginsight.dao.db.tables.BiUserTable; import org.breedinginsight.dao.db.tables.daos.ProgramDao; import org.breedinginsight.dao.db.tables.pojos.ProgramEntity; -import org.breedinginsight.dao.db.tables.records.ProgramRecord; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.User; import org.breedinginsight.model.*; @@ -394,6 +393,14 @@ public BrAPIClient getPhenoClient(UUID programId) { return client; } + @Override + public BrAPIClient getSampleClient(UUID programId) { + String brapiUrl = defaultBrAPIPhenoUrl; + BrAPIClient client = new BrAPIClient(brapiUrl); + initializeHttpClient(client); + return client; + } + private void initializeHttpClient(BrAPIClient brapiClient) { brapiClient.setHttpClient(brapiClient.getHttpClient() .newBuilder() diff --git a/src/main/java/org/breedinginsight/model/GenotypeVendor.java b/src/main/java/org/breedinginsight/model/GenotypeVendor.java new file mode 100644 index 000000000..e5302d32c --- /dev/null +++ b/src/main/java/org/breedinginsight/model/GenotypeVendor.java @@ -0,0 +1,37 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.model; + +public enum GenotypeVendor { + DART("DArT"); + + private final String name; + + GenotypeVendor(String name) { + this.name = name; + } + + public static GenotypeVendor fromName(String name) { + for(var vendor : GenotypeVendor.values()) { + if(vendor.name.equalsIgnoreCase(name)) { + return vendor; + } + } + return null; + } +} diff --git a/src/main/java/org/breedinginsight/model/SampleSubmission.java b/src/main/java/org/breedinginsight/model/SampleSubmission.java new file mode 100644 index 000000000..6202a5a6f --- /dev/null +++ b/src/main/java/org/breedinginsight/model/SampleSubmission.java @@ -0,0 +1,106 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import io.micronaut.core.annotation.Introspected; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; +import lombok.experimental.SuperBuilder; +import lombok.extern.jackson.Jacksonized; +import org.brapi.v2.model.geno.BrAPIPlate; +import org.brapi.v2.model.geno.BrAPISample; +import org.brapi.v2.model.geno.BrAPIShipmentForm; +import org.breedinginsight.dao.db.tables.pojos.SampleSubmissionEntity; +import org.jooq.JSONB; + +import java.util.List; + +@Getter +@Setter +@Accessors(chain=true) +@ToString +@SuperBuilder +@NoArgsConstructor +@Introspected +@Jacksonized +@JsonIgnoreProperties(value = {"createdBy", "updatedBy", "submittedBy", "shipmentforms"}) +public class SampleSubmission extends SampleSubmissionEntity { + private User createdByUser; + private User updatedByUser; + private User submittedByUser; + private List shipmentForms; + private List plates; + private List samples; + + public SampleSubmission(SampleSubmissionEntity entity) { + this.setId(entity.getId()); + this.setName(entity.getName()); + this.setSubmitted(entity.getSubmitted()); + this.setSubmittedDate(entity.getSubmittedDate()); + this.setSubmittedBy(entity.getSubmittedBy()); + this.setVendorOrderId(entity.getVendorOrderId()); + this.setVendorStatus(entity.getVendorStatus()); + this.setVendorStatusLastCheck(entity.getVendorStatusLastCheck()); + this.setShipmentforms(entity.getShipmentforms()); + this.setProgramId(entity.getProgramId()); + this.setCreatedAt(entity.getCreatedAt()); + this.setCreatedBy(entity.getCreatedBy()); + this.setUpdatedAt(entity.getUpdatedAt()); + this.setUpdatedBy(entity.getUpdatedBy()); + + parseShipmentForms(super.getShipmentforms()); + } + + private void parseShipmentForms(JSONB shipmentforms) { + if(shipmentforms != null) { + Gson gson = new Gson(); + this.shipmentForms = gson.fromJson(shipmentforms.data(), new TypeToken>() {}.getType()); + } + } + + public enum Status { + NOT_SUBMITTED("NOT SUBMITTED"), + SUBMITTED("SUBMITTED"), + COMPLETED("COMPLETED"); + + private final String value; + + Status(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + public static Status fromValue(String value) { + for(Status status : Status.values()) { + if(status.value.equals(value)) { + return status; + } + } + return null; + } + } +} diff --git a/src/main/java/org/breedinginsight/model/User.java b/src/main/java/org/breedinginsight/model/User.java index 15b38c1be..1aaf2283e 100644 --- a/src/main/java/org/breedinginsight/model/User.java +++ b/src/main/java/org/breedinginsight/model/User.java @@ -32,6 +32,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.UUID; import static org.breedinginsight.dao.db.Tables.BI_USER; @@ -43,7 +44,7 @@ @Introspected @Jacksonized @JsonIgnoreProperties(value = {"accountToken"}) -public class User extends BiUserEntity{ +public class User extends BiUserEntity { @JsonInclude() @NotNull diff --git a/src/main/java/org/breedinginsight/services/SampleSubmissionService.java b/src/main/java/org/breedinginsight/services/SampleSubmissionService.java new file mode 100644 index 000000000..2c629da55 --- /dev/null +++ b/src/main/java/org/breedinginsight/services/SampleSubmissionService.java @@ -0,0 +1,430 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.services; + +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Value; +import io.micronaut.scheduling.annotation.Scheduled; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.ApiResponse; +import org.brapi.client.v2.BrAPIClient; +import org.brapi.client.v2.auth.Authentication; +import org.brapi.client.v2.auth.OAuth; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.modules.genotype.VendorApi; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.geno.*; +import org.brapi.v2.model.geno.request.BrAPIVendorOrderSubmissionRequest; +import org.brapi.v2.model.geno.response.BrAPIVendorOrderStatusResponse; +import org.brapi.v2.model.geno.response.BrAPIVendorOrderStatusResponseResult; +import org.brapi.v2.model.geno.response.BrAPIVendorOrderSubmissionSingleResponse; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.daos.BrAPIPlateDAO; +import org.breedinginsight.brapps.importer.daos.BrAPISampleDAO; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.exports.FileType; +import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.SampleSubmissionDAO; +import org.breedinginsight.model.*; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.utilities.FileUtil; +import org.breedinginsight.utilities.Utilities; +import org.jooq.DSLContext; +import org.jooq.JSONB; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.io.IOException; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +@Singleton +public class SampleSubmissionService { + + private static final String COLUMN_GENOTYPE = "Genotype"; + private static final String VENDOR_NOT_SUBMITTED_STATUS = "NOT SUBMITTED"; + private static final String VENDOR_SUBMITTED_STATUS = "SUBMITTED"; + private final String referenceSource; + private final String dartBrapiUrl; + private final String dartClientId; + private final String dartToken; + private final Duration requestTimeout; + private final boolean brapiSubmissionEnabled; + + private final SampleSubmissionDAO submissionDAO; + private final BrAPIPlateDAO plateDAO; + private final BrAPISampleDAO sampleDAO; + private final BrAPIEndpointProvider brAPIEndpointProvider; + private final ProgramDAO programDAO; + private final DSLContext dsl; + + @Inject + public SampleSubmissionService(@Property(name = "brapi.server.reference-source") String referenceSource, + @Property(name = "brapi.vendors.dart.url") String dartBrapiUrl, + @Property(name = "brapi.vendors.dart.client-id") String dartClientId, + @Property(name = "brapi.vendors.dart.token") String dartToken, + @Value(value = "${brapi.read-timeout:5m}") Duration requestTimeout, + @Property(name = "brapi.vendor-submission-enabled") boolean brapiSubmissionEnabled, + SampleSubmissionDAO submissionDAO, + BrAPIPlateDAO plateDAO, + BrAPISampleDAO sampleDAO, + BrAPIEndpointProvider brAPIEndpointProvider, + ProgramDAO programDAO, + DSLContext dsl) { + this.referenceSource = referenceSource; + this.dartBrapiUrl = dartBrapiUrl; + this.dartClientId = dartClientId; + this.dartToken = dartToken; + this.requestTimeout = requestTimeout; + this.brapiSubmissionEnabled = brapiSubmissionEnabled; + this.submissionDAO = submissionDAO; + this.plateDAO = plateDAO; + this.sampleDAO = sampleDAO; + this.brAPIEndpointProvider = brAPIEndpointProvider; + this.programDAO = programDAO; + this.dsl = dsl; + } + + @Scheduled(fixedDelay = "${brapi.vendor-check-frequency}", initialDelay = "10s") + void checkSubmissionStatuses() { + if(!brapiSubmissionEnabled) { + return; + } + log.trace("checking vendor order statuses"); + List submittedAndNotCompleted = submissionDAO.getSubmittedAndNotComplete(); + log.trace(submittedAndNotCompleted.size() + " orders to check"); + for(var order : submittedAndNotCompleted) { + try { + this.checkVendorStatus(order); + } catch (ApiException e) { + log.error("Error checking vendor order status: \n\n" + Utilities.generateApiExceptionLogMessage(e), e); + throw new RuntimeException(e); + } + } + log.trace("vendor order status checks complete, sleeping"); + } + + public SampleSubmission createSubmission(SampleSubmission submission, Program program, ImportUpload upload) throws ApiException { + submission.setProgramId(program.getId()); + submission.setCreatedByUser(upload.getCreatedByUser()); + submission.setCreatedBy(upload.getCreatedBy()); + submission.setUpdatedByUser(upload.getUpdatedByUser()); + submission.setUpdatedBy(upload.getUpdatedBy()); + + dsl.transaction(() -> { + submissionDAO.insert(submission); + + List savedPlates = plateDAO.createPlates(program, submission.getPlates(), upload); + submission.setPlates(savedPlates); + Map plateNameToDbId = savedPlates.stream().collect(Collectors.toMap(BrAPIPlate::getPlateName, BrAPIPlate::getPlateDbId)); + + List samplesToSave = submission.getSamples().stream().map(sample -> sample.plateDbId(plateNameToDbId.get(sample.getPlateName()))).collect(Collectors.toList()); + List savedSamples = sampleDAO.createSamples(program, samplesToSave, upload); + submission.setSamples(savedSamples); + }); + + return submission; + } + + public Optional getSampleSubmission(Program program, UUID submissionId, boolean fetchDetails) throws ApiException { + if (fetchDetails) { + return populateSubmissions(program, submissionDAO.getBySubmissionId(program, submissionId)).stream().findFirst(); + } else { + return submissionDAO.getBySubmissionId(program, submissionId).stream().findFirst(); + } + } + + public List getProgramSubmissions(Program program) { + return submissionDAO.getByProgramId(program.getId()); + } + + private List populateSubmissions(Program program, List submissions) throws ApiException { + List submissionIds = submissions.stream().map(s -> s.getId().toString()).collect(Collectors.toList()); + List plates = plateDAO.readPlatesBySubmissionIds(program, submissionIds); + List samples = sampleDAO.readSamplesBySubmissionIds(program, submissionIds); + + Map> platesBySubmissionId = new HashMap<>(); + String submissionXrefSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATE_SUBMISSIONS); + plates.forEach(plate -> { + BrAPIExternalReference brAPIExternalReference = Utilities.getExternalReference(plate.getExternalReferences(), submissionXrefSource).orElseThrow(() -> new IllegalStateException(String.format("Plate %s does not have a submission ID", plate.getPlateName()))); + List submissionPlates = platesBySubmissionId.getOrDefault(brAPIExternalReference.getReferenceId(), new ArrayList<>()); + submissionPlates.add(plate); + platesBySubmissionId.putIfAbsent(brAPIExternalReference.getReferenceId(), submissionPlates); + }); + + Map> samplesBySubmissionId = new HashMap<>(); + samples.forEach(sample -> { + BrAPIExternalReference brAPIExternalReference = Utilities.getExternalReference(sample.getExternalReferences(), submissionXrefSource).orElseThrow(() -> new IllegalStateException(String.format("Plate %s does not have a submission ID", sample.getPlateName()))); + List submissionSamples = samplesBySubmissionId.getOrDefault(brAPIExternalReference.getReferenceId(), new ArrayList<>()); + submissionSamples.add(sample); + samplesBySubmissionId.putIfAbsent(brAPIExternalReference.getReferenceId(), submissionSamples); + }); + + submissions.forEach(submission -> { + submission.setPlates(platesBySubmissionId.get(submission.getId().toString())); + submission.setSamples(samplesBySubmissionId.get(submission.getId().toString())); + }); + + return submissions; + } + + public Optional generateDArTFile(Program program, UUID submissionId) throws ApiException, IOException { + Optional submission = getSampleSubmission(program, submissionId, true); + if (submission.isEmpty()) { + return Optional.empty(); + } + + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + String filename = Utilities.makePortableFilename(String.format("%s_DArT_Submission_%s.csv", submission.get().getName(), timestamp)); + + List columns = new ArrayList<>(); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.PLATE_ID).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.ROW).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.COLUMN).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.ORGANISM).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.SPECIES).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(COLUMN_GENOTYPE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.TISSUE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.COMMENTS).dataType(Column.ColumnDataType.STRING).build()); + + //TODO sort the samples first + List> rows = new ArrayList<>(); + submission.get().getSamples().forEach(sample -> { + Map row = new HashMap<>(); + row.put(SampleSubmissionImport.Columns.PLATE_ID, sample.getPlateName()); + row.put(SampleSubmissionImport.Columns.ROW, sample.getRow()); + row.put(SampleSubmissionImport.Columns.COLUMN, sample.getColumn()); + row.put(SampleSubmissionImport.Columns.ORGANISM, sample.getAdditionalInfo().get(BrAPIAdditionalInfoFields.SAMPLE_ORGANISM).getAsString()); + row.put(SampleSubmissionImport.Columns.SPECIES, sample.getAdditionalInfo().has(BrAPIAdditionalInfoFields.SAMPLE_SPECIES) ? sample.getAdditionalInfo().get(BrAPIAdditionalInfoFields.SAMPLE_SPECIES).getAsString() : ""); + row.put(COLUMN_GENOTYPE, sample.getSampleName()); + row.put(SampleSubmissionImport.Columns.TISSUE, sample.getTissueType()); + row.put(SampleSubmissionImport.Columns.COMMENTS, sample.getSampleDescription()); + + rows.add(row); + }); + + + return Optional.of(new DownloadFile(filename, FileUtil.writeToStreamedFile(columns, rows, FileType.CSV, "Data"))); + } + + public Optional generateLookupFile(Program program, UUID submissionId) throws ApiException, IOException { + Optional submission = getSampleSubmission(program, submissionId, true); + if (submission.isEmpty()) { + return Optional.empty(); + } + + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + String filename = Utilities.makePortableFilename(String.format("%s_Lookup_File_%s.csv", submission.get().getName(), timestamp)); + + List columns = new ArrayList<>(); + columns.add(Column.builder().value(COLUMN_GENOTYPE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.GERMPLASM_NAME).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.GERMPLASM_GID).dataType(Column.ColumnDataType.STRING).build()); + + //TODO sort the samples first + List> rows = new ArrayList<>(); + submission.get().getSamples().forEach(sample -> { + Map row = new HashMap<>(); + row.put(COLUMN_GENOTYPE, sample.getSampleName()); + row.put(SampleSubmissionImport.Columns.GERMPLASM_NAME, sample.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_NAME).getAsString()); + row.put(SampleSubmissionImport.Columns.GERMPLASM_GID, sample.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GID).getAsString()); + + rows.add(row); + }); + + + return Optional.of(new DownloadFile(filename, FileUtil.writeToStreamedFile(columns, rows, FileType.CSV, "Data"))); + } + + public Optional submitOrder(Program program, UUID submissionId, User submittingUser, GenotypeVendor vendor) throws ApiException, IllegalStateException { + Optional submissionOptional = getSampleSubmission(program, submissionId, true); + if (submissionOptional.isEmpty()) { + return Optional.empty(); + } + SampleSubmission submission = submissionOptional.get(); + + //prevent re-submission + if(Boolean.TRUE.equals(submission.getSubmitted())) { + if(StringUtils.isNotBlank(submission.getVendorOrderId())) { + return Optional.of(new BrAPIVendorOrderSubmission().orderId(submission.getVendorOrderId())); + } else { + throw new IllegalStateException("Submission has been manually submitted, cannot automatically submit to vendor"); + } + } + + Map platesForOrder = new HashMap<>(); + submission.getPlates().forEach(plate -> { + platesForOrder.put(plate.getPlateDbId(), new BrAPIVendorPlate() + .clientPlateBarcode(plate.getPlateBarcode()) + .clientPlateId(plate.getPlateName()) + .sampleSubmissionFormat(BrAPIPlateFormat.PLATE_96) + ); + }); + + submission.getSamples().forEach(sample -> { + BrAPIVendorPlate vendorPlate = platesForOrder.get(sample.getPlateDbId()); + vendorPlate.addSamplesItem(new BrAPIVendorSample() + .clientSampleId(sample.getSampleName()) + .clientSampleBarCode(sample.getSampleBarcode()) + .row(sample.getRow()) + .column(sample.getColumn()) + .comments(sample.getSampleDescription()) + .organismName(sample.getAdditionalInfo().get(BrAPIAdditionalInfoFields.SAMPLE_ORGANISM).getAsString()) + .speciesName(sample.getAdditionalInfo().has(BrAPIAdditionalInfoFields.SAMPLE_SPECIES) ? sample.getAdditionalInfo().get(BrAPIAdditionalInfoFields.SAMPLE_SPECIES).getAsString() : null) + .tissueType(sample.getTissueType()) + .well(String.format("%s%d", sample.getRow(), sample.getColumn())) + ); + }); + + //TODO get info for the specific vendor, and verify program has an account + BrAPIVendorOrderSubmissionRequest order = new BrAPIVendorOrderSubmissionRequest(); + VendorApi vendorApi; + if(GenotypeVendor.DART.equals(vendor)) { + order.setClientId(dartClientId); + vendorApi = getVendorApi(dartBrapiUrl, dartToken); + } else { + throw new IllegalStateException("Unrecognized vendor"); + } + + order.setNumberOfSamples(submission.getSamples().size()); + order.setPlates(new ArrayList<>(platesForOrder.values())); + + ApiResponse response = vendorApi.vendorOrdersPost(order); + + if (response.getBody() == null) { + throw new ApiException("Response is missing body", response.getStatusCode(), response.getHeaders(), null); + } + BrAPIVendorOrderSubmissionSingleResponse body = response.getBody(); + if (body.getResult() == null) { + throw new ApiException("Response body is missing result", response.getStatusCode(), response.getHeaders(), response.getBody().toString()); + } + BrAPIVendorOrderSubmission submittedOrder = body.getResult(); + + submission.setVendorOrderId(submittedOrder.getOrderId()); + submission.setVendorStatus(VENDOR_SUBMITTED_STATUS); + submission.setSubmitted(true); + submission.setSubmittedDate(OffsetDateTime.now()); + submission.setSubmittedByUser(submittingUser); + //TODO: TEMPORARY + submittedOrder.addShipmentFormsItem(new BrAPIShipmentForm().fileDescription("This is a shipment manifest form").fileName("Shipment Manifest").fileURL("https://vendor.org/forms/manifest.pdf")); + if(submittedOrder.getShipmentForms() != null) { + + submission.setShipmentforms(JSONB.valueOf(vendorApi.getApiClient().getJSON().serialize(submittedOrder.getShipmentForms()))); + } + + dsl.transaction(() -> { + submissionDAO.update(submission, submittingUser); + }); + + return Optional.of(submittedOrder); + } + + public Optional checkVendorStatus(Program program, UUID submissionId) throws ApiException { + Optional submissionOptional = getSampleSubmission(program, submissionId, true); + if (submissionOptional.isEmpty()) { + return Optional.empty(); + } + SampleSubmission submission = submissionOptional.get(); + + return Optional.of(checkVendorStatus(submission)); + } + + private SampleSubmission checkVendorStatus(SampleSubmission submission) throws ApiException { + if(submission.getVendorOrderId() == null || BrAPIVendorOrderStatusResponseResult.StatusEnum.COMPLETED.name().equalsIgnoreCase(submission.getVendorStatus())) { + return submission; + } + + VendorApi vendorApi = getVendorApi(dartBrapiUrl, dartToken); + ApiResponse response = vendorApi.vendorOrdersOrderIdStatusGet(submission.getVendorOrderId()); + if (response.getBody() == null) { + throw new ApiException("Response is missing body", response.getStatusCode(), response.getHeaders(), null); + } + BrAPIVendorOrderStatusResponse body = response.getBody(); + if (body.getResult() == null) { + throw new ApiException("Response body is missing result", response.getStatusCode(), response.getHeaders(), response.getBody().toString()); + } + BrAPIVendorOrderStatusResponseResult result = body.getResult(); + + if(result.getStatus() != null) { + submission.setVendorStatus(result.getStatus().name()); + } + submission.setVendorStatusLastCheck(OffsetDateTime.now()); + + dsl.transaction(() -> { + submissionDAO.update(submission, submission.getUpdatedByUser()); + }); + + return submission; + } + + private VendorApi getVendorApi(String url, String authToken) { + BrAPIClient client = new BrAPIClient(url); + client.setHttpClient(client.getHttpClient() + .newBuilder() + .readTimeout(requestTimeout) + .build()); + + Authentication authorizationToken = client.getAuthentication("AuthorizationToken"); + if(authorizationToken instanceof OAuth) { + ((OAuth)authorizationToken).setAccessToken(authToken); + } + + return brAPIEndpointProvider.get(client, VendorApi.class); + } + + public Optional updateSubmissionStatus(Program program, UUID submissionId, SampleSubmission.Status status, User user) throws ApiException { + Optional submissionOptional = this.getSampleSubmission(program, submissionId, false); + if(submissionOptional.isEmpty()) { + return Optional.empty(); + } + + SampleSubmission submission = submissionOptional.get(); + if(StringUtils.isBlank(submission.getVendorOrderId())) { + SampleSubmission.Status currentStatus = SampleSubmission.Status.fromValue(submission.getVendorStatus()); + if(currentStatus != status) { + if((currentStatus == null || currentStatus == SampleSubmission.Status.NOT_SUBMITTED) && status != SampleSubmission.Status.NOT_SUBMITTED) { + submission.setSubmitted(true); + submission.setSubmittedByUser(user); + submission.setSubmittedDate(OffsetDateTime.now()); + } + submission.setVendorStatus(status.getValue()); + + if(status == SampleSubmission.Status.NOT_SUBMITTED){ + submission.setSubmitted(false); + submission.setSubmittedBy(null); + submission.setSubmittedByUser(null); + submission.setSubmittedDate(null); + submission.setVendorStatus(null); + } + + return Optional.ofNullable(submissionDAO.update(submission, user)); + } + } + + return submissionOptional; + } +} diff --git a/src/main/java/org/breedinginsight/services/TraitUploadService.java b/src/main/java/org/breedinginsight/services/TraitUploadService.java index 4534c095f..2caf03bcc 100644 --- a/src/main/java/org/breedinginsight/services/TraitUploadService.java +++ b/src/main/java/org/breedinginsight/services/TraitUploadService.java @@ -111,7 +111,8 @@ public ProgramUpload updateTraitUpload(UUID programId, CompletedFileUpload file, throw new HttpStatusException(HttpStatus.BAD_REQUEST, "Error parsing csv: " + e.getMessage()); } } else if (mediaType.toString().equals(SupportedMediaType.XLS) || - mediaType.toString().equals(SupportedMediaType.XLSX)) { + mediaType.toString().equals(SupportedMediaType.XLSX) || + mediaType.toString().equals(SupportedMediaType.XLSB)) { try { traits = parser.parseExcel(new BOMInputStream(file.getInputStream(), false)); } catch(IOException | ParsingException e) { diff --git a/src/main/java/org/breedinginsight/services/constants/SupportedMediaType.java b/src/main/java/org/breedinginsight/services/constants/SupportedMediaType.java index 34de108c0..4f42bacd1 100644 --- a/src/main/java/org/breedinginsight/services/constants/SupportedMediaType.java +++ b/src/main/java/org/breedinginsight/services/constants/SupportedMediaType.java @@ -22,4 +22,6 @@ public class SupportedMediaType { public static final String XLS = "application/vnd.ms-excel"; public static final String XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + public static final String XLSB = "application/vnd.ms-excel.sheet.binary.macroenabled.12"; + } diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index e0c470c9b..e03a67017 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -281,12 +281,12 @@ public List put(List brapiObjects, throw new UnsupportedOperationException(); } - public List post(List brapiObjects, - ImportUpload upload, - Function, ApiResponse> postMethod, - Consumer progressUpdateMethod) throws ApiException { + public List post(List brapiObjects, + ImportUpload upload, + Function, ApiResponse> postMethod, + Consumer progressUpdateMethod) throws ApiException { - List listResult = new ArrayList<>(); + List listResult = new ArrayList<>(); try { // Make the POST calls in chunks so we don't overload the brapi server Integer currentRightBorder = 0; @@ -314,7 +314,7 @@ public List post(List brapiObjects, if (result.getData() == null) { throw new ApiException("Response result is missing data", response.getStatusCode(), response.getHeaders(), response.getBody().toString()); } - List data = result.getData(); + List data = result.getData(); // TODO: Maybe move this outside of the loop if (data.size() != postChunk.size()) { throw new ApiException("Number of brapi objects returned does not equal number sent"); diff --git a/src/main/java/org/breedinginsight/utilities/FileUtil.java b/src/main/java/org/breedinginsight/utilities/FileUtil.java index 09a2962fa..985103bc1 100644 --- a/src/main/java/org/breedinginsight/utilities/FileUtil.java +++ b/src/main/java/org/breedinginsight/utilities/FileUtil.java @@ -17,14 +17,18 @@ package org.breedinginsight.utilities; +import io.micronaut.http.server.types.files.StreamedFile; import lombok.extern.slf4j.Slf4j; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.io.input.BOMInputStream; import org.apache.poi.EncryptedDocumentException; import org.apache.poi.ss.usermodel.*; +import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.services.parsers.ParsingException; import org.breedinginsight.services.parsers.ParsingExceptionType; +import org.breedinginsight.services.writers.CSVWriter; +import org.breedinginsight.services.writers.ExcelWriter; import tech.tablesaw.api.ColumnType; import tech.tablesaw.api.StringColumn; import tech.tablesaw.api.Table; @@ -246,4 +250,12 @@ public static Table removeNullColumns(Table table) throws ParsingException { return table; } + + public static StreamedFile writeToStreamedFile(List columns, List> data, FileType extension, String sheetName) throws IOException { + if (extension.equals(FileType.CSV)){ + return CSVWriter.writeToDownload(columns, data, extension); + } else { + return ExcelWriter.writeToDownload(sheetName, columns, data, extension); + } + } } diff --git a/src/main/resources/application-dev.yml.template b/src/main/resources/application-dev.yml.template new file mode 100644 index 000000000..81d9add14 --- /dev/null +++ b/src/main/resources/application-dev.yml.template @@ -0,0 +1,10 @@ +##For local development: +## 1. spin up a LocalStack container +## 2. copy this file and rename it to `application-dev.yml` +## 3. uncomment these lines, and update the `endpoint` value +## 4. set the environment variable MICRONAUT_ENVIRONMENTS=dev +#aws: +# s3: +# buckets: +# geno: +# endpoint: \ No newline at end of file diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 5b3e50999..aec211b82 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -154,12 +154,19 @@ web: brapi: server: - default-url: ${BRAPI_DEFAULT_URL:`https://test-server.brapi.org`} + default-url: ${BRAPI_DEFAULT_URL} # leave these for future but all point to default for now core-url: ${brapi.server.default-url} pheno-url: ${brapi.server.default-url} geno-url: ${brapi.server.default-url} reference-source: ${BRAPI_REFERENCE_SOURCE:breedinginsight.org} + vendor-submission-enabled: ${BRAPI_VENDOR_SUBMISSION_ENABLED:false} + vendor-check-frequency: ${BRAPI_VENDOR_CHECK_FREQUENCY:1d} + vendors: + dart: + url: ${DART_VENDOR_URL:`https://test-server.brapi.org`} + client-id: ${DART_CLIENT_ID:potato-salad} + token: ${DART_TOKEN:YYYY} read-timeout: ${BRAPI_READ_TIMEOUT:10m} page-size: 1000 search: diff --git a/src/main/resources/db/migration/V1.16.0__create_sampleimport_mapping.sql b/src/main/resources/db/migration/V1.16.0__create_sampleimport_mapping.sql new file mode 100644 index 000000000..50bac1903 --- /dev/null +++ b/src/main/resources/db/migration/V1.16.0__create_sampleimport_mapping.sql @@ -0,0 +1,108 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +DO +$$ + DECLARE + user_id UUID; + BEGIN + + user_id := (SELECT id FROM bi_user WHERE name = 'system'); + + INSERT INTO public.importer_mapping (id, name, import_type_id, mapping, file, draft, created_at, updated_at, created_by, updated_by) + VALUES (uuid_generate_v4(), + 'SampleImport', + 'SampleImport', '[ + { + "id": "e4038afd-02b9-475a-88b2-4692e331c399", + "value": { + "fileFieldName": "PlateID" + }, + "objectId": "plateId" + }, + { + "id": "9488023c-ed28-4357-811e-7c517cbd9f68", + "value": { + "fileFieldName": "Row" + }, + "objectId": "row" + }, + { + "id": "54b47821-c868-4fbd-acd6-8b8a64a02230", + "value": { + "fileFieldName": "Column" + }, + "objectId": "column" + }, + { + "id": "f9e39005-e8be-4e83-92b8-a459fe296d2f", + "value": { + "fileFieldName": "Organism" + }, + "objectId": "organism" + }, + { + "id": "fe37a2a5-00e0-4076-b130-846f19b3defd", + "value": { + "fileFieldName": "Species" + }, + "objectId": "species" + }, + { + "id": "e5074e78-b8ba-474e-87df-716a759f0517", + "value": { + "fileFieldName": "Germplasm Name" + }, + "objectId": "germplasmName" + }, + { + "id": "15a20991-ee85-49c6-b5c3-5db4e3dca372", + "value": { + "fileFieldName": "GID" + }, + "objectId": "gid" + }, + { + "id": "11e2af68-ca45-4164-9bbb-326c68894fec", + "value": { + "fileFieldName": "ObsUnitID" + }, + "objectId": "obsUnitId" + }, + { + "id": "e7731381-7b92-42c3-97f0-38fb37208e8d", + "value": { + "fileFieldName": "Tissue" + }, + "objectId": "tissue" + }, + { + "id": "de7fed3a-ec44-4139-83cb-c773e09237bd", + "value": { + "fileFieldName": "Comments" + }, + "objectId": "comments" + } + ]', '[]', + false, + '2023-10-18 02:03:17 +00:00', + '2023-10-18 02:03:17 +00:00', + user_id, + user_id); + + END +$$; \ No newline at end of file diff --git a/src/main/resources/db/migration/V1.17.0__create_sample_submission_table.sql b/src/main/resources/db/migration/V1.17.0__create_sample_submission_table.sql new file mode 100644 index 000000000..71c9561b2 --- /dev/null +++ b/src/main/resources/db/migration/V1.17.0__create_sample_submission_table.sql @@ -0,0 +1,40 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +create table sample_submission +( + like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, + name TEXT, + submitted bool default false, + submitted_date timestamp(0) with time zone, + submitted_by UUID, + vendor_order_id TEXT, + vendor_status TEXT, + vendor_status_last_check timestamp(0) with time zone, + shipmentforms jsonb, + program_id UUID NOT NULL, + like base_edit_track_entity INCLUDING ALL +); + +ALTER TABLE sample_submission + ADD FOREIGN KEY (created_by) REFERENCES bi_user (id); +ALTER TABLE sample_submission + ADD FOREIGN KEY (updated_by) REFERENCES bi_user (id); +ALTER TABLE sample_submission + ADD FOREIGN KEY (program_id) REFERENCES program (id); +ALTER TABLE sample_submission + ADD FOREIGN KEY (submitted_by) REFERENCES bi_user (id); \ No newline at end of file diff --git a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java new file mode 100644 index 000000000..a73cc1ac8 --- /dev/null +++ b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java @@ -0,0 +1,498 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.v1.controller; + +import com.eclipsesource.json.Json; +import com.google.gson.*; +import com.google.gson.reflect.TypeToken; +import io.kowalski.fannypack.FannyPack; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.netty.cookies.NettyCookie; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import io.reactivex.Flowable; +import org.apache.commons.lang3.tuple.Pair; +import org.brapi.client.v2.BrAPIClient; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.geno.BrAPISample; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.TestUtils; +import org.breedinginsight.api.model.v1.request.ProgramRequest; +import org.breedinginsight.api.model.v1.request.SpeciesRequest; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.ImportTestUtils; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport; +import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport.Columns; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.dao.db.tables.pojos.ProgramBreedingMethodEntity; +import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.SpeciesDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.*; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.parsers.ParsingException; +import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; +import org.breedinginsight.services.writers.CSVWriter; +import org.breedinginsight.utilities.FileUtil; +import org.breedinginsight.utilities.Utilities; +import org.jetbrains.annotations.NotNull; +import org.jooq.DSLContext; +import org.junit.jupiter.api.*; +import org.testcontainers.containers.localstack.LocalStackContainer; +import tech.tablesaw.api.Table; + +import javax.inject.Inject; +import java.io.*; +import java.time.OffsetDateTime; +import java.util.*; + +import static io.micronaut.http.HttpRequest.*; +import static org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields.SUBMISSION_NAME; +import static org.junit.jupiter.api.Assertions.*; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class SampleSubmissionControllerIntegrationTest extends BrAPITest { + + private Program program; + + private ImportTestUtils importTestUtils; + + private String experimentId; + private List envIds = new ArrayList<>(); + private final List> rows = new ArrayList<>(); + private final List columns = ExperimentFileColumns.getOrderedColumns(); + private List traits; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + @Inject + private DSLContext dsl; + @Inject + private UserDAO userDAO; + @Inject + private SpeciesDAO speciesDAO; + @Inject + private OntologyService ontologyService; + @Inject + private BrAPIGermplasmDAO germplasmDAO; + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + private final Gson gson = new BrAPIClient().getJSON().getGson(); + + @BeforeAll + void setup() throws Exception { + importTestUtils = new ImportTestUtils(); + FannyPack fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); + FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); + + // Test User + User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + // Species + super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + + // Test Program + ProgramRequest programRequest = ProgramRequest.builder() + .name("Test Program") + .abbreviation("Test") + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key("TEST") + .build(); + program = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + List germplasm = createGermplasm(96); + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); + newReference.setReferenceID(program.getId().toString()); + + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); + + germplasmDAO.createBrAPIGermplasm(germplasm, program.getId(), null); + } + + @NotNull + @Override + public Map getProperties() { + Map properties = super.getProperties(); + + properties.put("brapi.vendor-submission-enabled", "true"); + + Integer containerPort = getBrapiContainer().getMappedPort(8080); + String containerIp = getBrapiContainer().getContainerIpAddress(); + properties.put("brapi.vendors.dart.url", String.format("http://%s:%s/", containerIp, containerPort)); + + return properties; + } + + /* + Tests + - fetch all sample submissions + - fetch individual sample submission + - manual update submission status + - brapi submit + - check vendor status + + */ + + @Test + public void testFetchProgramSubmissions() throws IOException, InterruptedException { + List submissionIds = new ArrayList<>(); + submissionIds.add(createSubmission(program).getLeft().getId()); + submissionIds.add(createSubmission(program).getLeft().getId()); + + Flowable> call = client.exchange( + GET(String.format("/programs/%s/submissions", program.getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + List submissions = gson.fromJson(JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").getAsJsonArray("data"), new TypeToken>(){}.getType()); + assertTrue(submissions.size() >= 2); + + + List returnedSubmissionIds = new ArrayList<>(); + for(var submission : submissions) { + if(submissionIds.contains(submission.getId())) { + returnedSubmissionIds.add(submission.getId()); + } + } + assertEquals(submissionIds.size(), returnedSubmissionIds.size()); + } + + @Test + public void testFetchIndividualSubmissions() throws IOException, InterruptedException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> call = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=true", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + SampleSubmission retrievedSubmission = gson.fromJson(JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + + assertNotNull(retrievedSubmission); + assertEquals(uploadedSubmission.getLeft().getId(), retrievedSubmission.getId()); + assertEquals(uploadedSubmission.getLeft().getName(), retrievedSubmission.getName()); + assertEquals(1, retrievedSubmission.getPlates().size()); + assertEquals(96, retrievedSubmission.getSamples().size()); + } + + @Test + public void testManualUpdateSubmissions() throws IOException, InterruptedException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> putCall = client.exchange( + PUT(String.format("/programs/%s/submissions/%s/status", program.getId(), uploadedSubmission.getLeft().getId()), "{status:\"SUBMITTED\"}").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse putResponse = putCall.blockingFirst(); + assertNotNull(putResponse.body()); + + Flowable> fetchCall = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=true", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse fetchResponse = fetchCall.blockingFirst(); + SampleSubmission retrievedSubmission = gson.fromJson(JsonParser.parseString(fetchResponse.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + + assertNotNull(retrievedSubmission); + assertEquals("SUBMITTED", retrievedSubmission.getVendorStatus()); + assertNull(retrievedSubmission.getVendorOrderId()); + assertNull(retrievedSubmission.getVendorStatusLastCheck()); + } + + @Test + public void testSubmitViaBrAPI() throws IOException, InterruptedException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> postCall = client.exchange( + POST(String.format("/programs/%s/submissions/%s/submit?vendor=dart", program.getId(), uploadedSubmission.getLeft().getId()), null).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse postResponse = postCall.blockingFirst(); + assertNotNull(postResponse.body()); + + Flowable> fetchCall = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=true", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse fetchResponse = fetchCall.blockingFirst(); + SampleSubmission retrievedSubmission = gson.fromJson(JsonParser.parseString(fetchResponse.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + + assertNotNull(retrievedSubmission); + assertEquals("SUBMITTED", retrievedSubmission.getVendorStatus()); + assertNotNull(retrievedSubmission.getVendorOrderId()); + assertNull(retrievedSubmission.getVendorStatusLastCheck()); + } + + @Test + public void testCheckVendorStatus() throws IOException, InterruptedException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> postCall = client.exchange( + POST(String.format("/programs/%s/submissions/%s/submit?vendor=dart", program.getId(), uploadedSubmission.getLeft().getId()), null).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse postResponse = postCall.blockingFirst(); + assertNotNull(postResponse.body()); + + Flowable> fetchCall = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=false", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse fetchResponse = fetchCall.blockingFirst(); + SampleSubmission retrievedSubmission = gson.fromJson(JsonParser.parseString(fetchResponse.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + + assertNotNull(retrievedSubmission); + assertNotNull(retrievedSubmission.getVendorOrderId()); + assertNull(retrievedSubmission.getVendorStatusLastCheck()); + + Flowable> fetchStatus = client.exchange( + GET(String.format("/programs/%s/submissions/%s/status", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse fetchStatusResponse = fetchStatus.blockingFirst(); + assertNotNull(fetchStatusResponse.body()); + + Flowable> fetchSubmissionAfterStatus = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=false", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse fetchSubmissionAfterStatusResponse = fetchSubmissionAfterStatus.blockingFirst(); + SampleSubmission updatedStatusResponse = gson.fromJson(JsonParser.parseString(fetchSubmissionAfterStatusResponse.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + assertNotNull(updatedStatusResponse.getVendorStatusLastCheck()); + + Thread.sleep(1000); + + fetchStatus = client.exchange( + GET(String.format("/programs/%s/submissions/%s/status", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + fetchStatusResponse = fetchStatus.blockingFirst(); + assertNotNull(fetchStatusResponse.body()); + + fetchSubmissionAfterStatus = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=false", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + fetchSubmissionAfterStatusResponse = fetchSubmissionAfterStatus.blockingFirst(); + SampleSubmission updatedStatusResponse2 = gson.fromJson(JsonParser.parseString(fetchSubmissionAfterStatusResponse.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + assertTrue(updatedStatusResponse2.getVendorStatusLastCheck().isAfter(updatedStatusResponse.getVendorStatusLastCheck())); + } + + @Test + public void testGenerateDArTFile() throws IOException, InterruptedException, ParsingException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> call = client.exchange( + GET(String.format("/programs/%s/submissions/%s/dart", + program.getId().toString(), uploadedSubmission.getLeft().getId())) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse response = call.blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + + ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(response.body())); + Table lookupTable = FileUtil.parseTableFromCsv(bodyStream); + assertEquals(8, lookupTable.columnCount()); + assertEquals(Columns.PLATE_ID, lookupTable.column(0).name()); + assertEquals(Columns.ROW, lookupTable.column(1).name()); + assertEquals(Columns.COLUMN, lookupTable.column(2).name()); + assertEquals(Columns.ORGANISM, lookupTable.column(3).name()); + assertEquals(Columns.SPECIES, lookupTable.column(4).name()); + assertEquals("Genotype", lookupTable.column(5).name()); + assertEquals(Columns.TISSUE, lookupTable.column(6).name()); + assertEquals(Columns.COMMENTS, lookupTable.column(7).name()); + } + + @Test + public void testGenerateLookupFile() throws IOException, InterruptedException, ParsingException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> call = client.exchange( + GET(String.format("/programs/%s/submissions/%s/lookup", + program.getId().toString(), uploadedSubmission.getLeft().getId())) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse response = call.blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + + ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(response.body())); + Table lookupTable = FileUtil.parseTableFromCsv(bodyStream); + assertEquals(3, lookupTable.columnCount()); + assertEquals("Genotype", lookupTable.column(0).name()); + assertEquals("Germplasm Name", lookupTable.column(1).name()); + assertEquals("GID", lookupTable.column(2).name()); + } + + + private Pair>> createSubmission(Program program) throws IOException, InterruptedException { + Flowable> call = client.exchange( + GET("/import/mappings?importName=SampleImport").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + String sampleMappingId = JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + var submissionData = makeSubmission(); + var submission = new SampleSubmission(); + submission.setName("test-"+UUID.randomUUID()); + + JsonObject importResult = importTestUtils.uploadAndFetch( + writeSubmissionToFile(submissionData), + Map.of(SUBMISSION_NAME, submission.getName()), + true, + client, + program, + sampleMappingId); + JsonArray previewRows = importResult.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(96, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + BrAPISample sample = new Gson().fromJson(row.getAsJsonObject("sample").getAsJsonObject("brAPIObject"), BrAPISample.class); + BrAPIExternalReference xref = Utilities.getExternalReference(sample.getExternalReferences(), Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.PLATE_SUBMISSIONS)).get(); + submission.setId(UUID.fromString(xref.getReferenceId())); + + return Pair.of(submission, submissionData); + } + + private List> makeSubmission() { + List> validFile = new ArrayList<>(); + int germGidCounter = 1; + for(int i = 0; i < 8; i++) { + for(int j = 0; j < 12; j++) { + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A' + i)); + validRow.put(Columns.COLUMN, j+1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, germGidCounter++); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + } + } + + return validFile; + } + + private String createExperiment(Program program) throws IOException, InterruptedException { + Flowable> call = client.exchange( + GET("/import/mappings?importName=ExperimentsTemplateMap").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + String expMappingId = JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + JsonObject importResult = importTestUtils.uploadAndFetch( + importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null), + null, + true, + client, + program, + expMappingId); + return importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + } + + private Map makeExpImportRow(String environment) { + Map row = new HashMap<>(); + row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); + row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); + row.put(ExperimentObservation.Columns.EXP_TITLE, "Test Exp"); + row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); + row.put(ExperimentObservation.Columns.ENV, environment); + row.put(ExperimentObservation.Columns.ENV_LOCATION, "Location A"); + row.put(ExperimentObservation.Columns.ENV_YEAR, "2023"); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, "a-1"); + row.put(ExperimentObservation.Columns.REP_NUM, "1"); + row.put(ExperimentObservation.Columns.BLOCK_NUM, "1"); + row.put(ExperimentObservation.Columns.ROW, "1"); + row.put(ExperimentObservation.Columns.COLUMN, "1"); + return row; + } + + public File writeSubmissionToFile(List> data) throws IOException { + File file = File.createTempFile("test", ".csv"); + + List columns = new ArrayList<>(); + columns.add(Column.builder().value(Columns.PLATE_ID).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.ROW).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.COLUMN).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.ORGANISM).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.SPECIES).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.GERMPLASM_NAME).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.GERMPLASM_GID).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.OBS_UNIT_ID).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.TISSUE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.COMMENTS).dataType(Column.ColumnDataType.STRING).build()); + + ByteArrayOutputStream byteArrayOutputStream = CSVWriter.writeToCSV(columns, data); + FileOutputStream fos = new FileOutputStream(file); + fos.write(byteArrayOutputStream.toByteArray()); + + return file; + } + + private List createGermplasm(int numToCreate) { + List germplasm = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String gid = ""+(i+1); + BrAPIGermplasm testGermplasm = new BrAPIGermplasm(); + testGermplasm.setGermplasmName(String.format("Germplasm %s [TEST-%s]", gid, gid)); + testGermplasm.setSeedSource("Wild"); + testGermplasm.setAccessionNumber(gid); + testGermplasm.setDefaultDisplayName(String.format("Germplasm %s", gid)); + JsonObject additionalInfo = new JsonObject(); + additionalInfo.addProperty("importEntryNumber", gid); + additionalInfo.addProperty("breedingMethod", "Allopolyploid"); + testGermplasm.setAdditionalInfo(additionalInfo); + List externalRef = new ArrayList<>(); + BrAPIExternalReference testReference = new BrAPIExternalReference(); + testReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); + testReference.setReferenceID(UUID.randomUUID().toString()); + externalRef.add(testReference); + testGermplasm.setExternalReferences(externalRef); + germplasm.add(testGermplasm); + } + + return germplasm; + } +} diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java index d9c05ed21..ebb3979d1 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java @@ -387,6 +387,7 @@ private BrAPIObservationVariable generateVariable() { return new BrAPIObservationVariable().observationVariableName("test" + random) .commonCropName("Grape") .externalReferences(Collections.singletonList(new BrAPIExternalReference().referenceID("abc123") + .referenceId("abc123") .referenceSource("breedinginsight.org"))) .trait(new BrAPITrait().traitClass("Agronomic") .traitName("test trait" + random)) diff --git a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java index ccdf04e19..fdabb53db 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java @@ -142,7 +142,7 @@ public void setup() { newExp.put(traits.get(0).getObservationVariableName(), "1"); JsonObject result = importTestUtils.uploadAndFetch( - importTestUtils.writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId + importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId ); } diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index ac8520a68..748299daf 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -18,7 +18,6 @@ package org.breedinginsight.brapps.importer; import com.google.gson.*; -import com.google.gson.reflect.TypeToken; import io.kowalski.fannypack.FannyPack; import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpResponse; @@ -50,7 +49,6 @@ import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; import org.breedinginsight.daos.ProgramDAO; @@ -64,7 +62,6 @@ import org.breedinginsight.services.exceptions.BadRequestException; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.ValidatorException; -import org.breedinginsight.services.writers.CSVWriter; import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; import org.junit.jupiter.api.*; @@ -74,9 +71,7 @@ import org.opentest4j.AssertionFailedError; import javax.inject.Inject; -import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.time.OffsetDateTime; import java.util.*; @@ -199,7 +194,7 @@ public void importNewExpNewLocNoObsSuccess() { validRow.put(Columns.COLUMN, "1"); validRow.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeDataToFile(List.of(validRow), null), null, true, client, program, mappingId); + Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); @@ -257,7 +252,7 @@ public void importNewExpMultiNewEnvNoObsSuccess() { secondEnv.put(Columns.COLUMN, "1"); secondEnv.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); + Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); @@ -304,7 +299,7 @@ public void importNewEnvExistingExpNoObsSuccess() { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); Map newEnv = new HashMap<>(); newEnv.put(Columns.GERMPLASM_GID, "1"); @@ -321,7 +316,7 @@ public void importNewEnvExistingExpNoObsSuccess() { newEnv.put(Columns.ROW, "1"); newEnv.put(Columns.COLUMN, "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -355,43 +350,43 @@ public void verifyMissingDataThrowsError(boolean commit) { Map noGID = new HashMap<>(base); noGID.remove(Columns.GERMPLASM_GID); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); Map noExpTitle = new HashMap<>(base); noExpTitle.remove(Columns.EXP_TITLE); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); Map noExpUnit = new HashMap<>(base); noExpUnit.remove(Columns.EXP_UNIT); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); Map noExpType = new HashMap<>(base); noExpType.remove(Columns.EXP_TYPE); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); Map noEnv = new HashMap<>(base); noEnv.remove(Columns.ENV); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noEnv), null), Columns.ENV, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit); Map noEnvLoc = new HashMap<>(base); noEnvLoc.remove(Columns.ENV_LOCATION); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); Map noExpUnitId = new HashMap<>(base); noExpUnitId.remove(Columns.EXP_UNIT_ID); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); Map noExpRep = new HashMap<>(base); noExpRep.remove(Columns.REP_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); Map noExpBlock = new HashMap<>(base); noExpBlock.remove(Columns.BLOCK_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); Map noEnvYear = new HashMap<>(base); noEnvYear.remove(Columns.ENV_YEAR); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); } @Test @@ -415,7 +410,7 @@ public void importNewExpWithObsVar() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -465,7 +460,7 @@ public void verifyDiffYearSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(rows, null), Columns.ENV_YEAR, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit); } @ParameterizedTest @@ -503,7 +498,7 @@ public void verifyDiffLocSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(rows, null), Columns.ENV_LOCATION, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit); } @ParameterizedTest @@ -528,7 +523,7 @@ public void importNewExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -568,7 +563,7 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "Red"); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); } @ParameterizedTest @@ -591,14 +586,14 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); Map newOU = new HashMap<>(newExp); newOU.put(Columns.EXP_UNIT_ID, "a-2"); newOU.put(Columns.ROW, "1"); newOU.put(Columns.COLUMN, "2"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); + Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); @@ -632,7 +627,7 @@ public void importNewObsVarExisingOu() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -660,7 +655,7 @@ public void importNewObsVarExisingOu() { newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); newObsVar.put(traits.get(1).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -697,7 +692,7 @@ public void importNewObsExisingOu(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -725,7 +720,7 @@ public void importNewObsExisingOu(boolean commit) { newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -764,7 +759,7 @@ public void verifyFailureImportNewObsExisingOuWithExistingObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -792,7 +787,7 @@ public void verifyFailureImportNewObsExisingOuWithExistingObs(boolean commit) { newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); newObservation.put(traits.get(0).getObservationVariableName(), "2"); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit); } /* @@ -822,7 +817,7 @@ public void importSecondExpAfterFirstExpWithObs() { newExpA.put(Columns.COLUMN, "1"); newExpA.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); JsonArray previewRowsA = resultA.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsA.size()); @@ -850,7 +845,7 @@ public void importSecondExpAfterFirstExpWithObs() { newExpB.put(Columns.COLUMN, "1"); newExpB.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); JsonArray previewRowsB = resultB.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsB.size()); @@ -891,7 +886,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -920,7 +915,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -965,7 +960,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -996,7 +991,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newObservation.put(traits.get(0).getObservationVariableName(), ""); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index b8b00374d..3a521aa55 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -196,7 +196,7 @@ public List createTraits(int numToCreate) { return traits; } - public File writeDataToFile(List> data, List traits) throws IOException { + public File writeExperimentDataToFile(List> data, List traits) throws IOException { File file = File.createTempFile("test", ".csv"); List columns = new ArrayList<>(); diff --git a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java new file mode 100644 index 000000000..aa446cc55 --- /dev/null +++ b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java @@ -0,0 +1,592 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer; + +import com.google.gson.*; +import io.kowalski.fannypack.FannyPack; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.netty.cookies.NettyCookie; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import io.reactivex.Flowable; +import lombok.SneakyThrows; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.typeAdapters.PaginationTypeAdapter; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.BrAPIPagination; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.geno.BrAPIPlate; +import org.brapi.v2.model.geno.BrAPISample; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.TestUtils; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramRequest; +import org.breedinginsight.api.model.v1.request.SpeciesRequest; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.daos.*; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport.Columns; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; +import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.SpeciesDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.Column; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.SampleSubmission; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.*; +import org.breedinginsight.services.exceptions.BadRequestException; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.services.writers.CSVWriter; +import org.breedinginsight.utilities.Utilities; +import org.jooq.DSLContext; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.junit.platform.commons.util.StringUtils; + +import javax.inject.Inject; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.*; + +import static io.micronaut.http.HttpRequest.GET; +import static org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields.SUBMISSION_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class SampleSubmissionFileImportTest extends BrAPITest { + + private FannyPack securityFp; + private String mappingId; + private BiUserEntity testUser; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + @Property(name = "brapi.server.core-url") + private String BRAPI_URL; + + @Inject + private SpeciesService speciesService; + @Inject + private UserDAO userDAO; + @Inject + private DSLContext dsl; + + @Inject + private SpeciesDAO speciesDAO; + + @Inject + private ProgramService programService; + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + private ImportTestUtils importTestUtils; + + @Inject + private OntologyService ontologyService; + + @Inject + private BrAPITrialDAO brAPITrialDAO; + + @Inject + private BrAPIStudyDAO brAPIStudyDAO; + + @Inject + private BrAPIObservationUnitDAO ouDAO; + + @Inject + private ProgramLocationService locationService; + + @Inject + private BrAPIGermplasmDAO germplasmDAO; + + @Inject + private BrAPIObservationDAO observationDAO; + + @Inject + private BrAPISeasonDAO seasonDAO; + + @Inject + private SampleSubmissionService sampleSubmissionService; + + private Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .registerTypeAdapter(BrAPIPagination.class, new PaginationTypeAdapter()) + .create(); + + @BeforeAll + public void setup() { + importTestUtils = new ImportTestUtils(); + Map setupObjects = importTestUtils.setup(client, gson, dsl, speciesService, userDAO, super.getBrapiDsl(), "SampleImport"); + mappingId = (String) setupObjects.get("mappingId"); + testUser = (BiUserEntity) setupObjects.get("testUser"); + securityFp = (FannyPack) setupObjects.get("securityFp"); + + } + + /* + Tests + - valid submission GID + - valid submission ObsUnitID + - missing columns error + - conflicting wells error + - bad GID error + - bad ObsUnitID error + */ + + @Test + @SneakyThrows + public void importGIDSuccess() { + Program program = createProgram("Import GID Success", "GIDS", "GIDS", BRAPI_REFERENCE_SOURCE, createGermplasm(96), null); + List> validFile = new ArrayList<>(); + + int germGidCounter = 1; + for(int i = 0; i < 8; i++) { + for(int j = 0; j < 12; j++) { + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A' + i)); + validRow.put(Columns.COLUMN, j+1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, germGidCounter++); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + } + } + + Flowable> call = importTestUtils.uploadDataFile(writeDataToFile(validFile), Map.of(SUBMISSION_NAME, "test-"+UUID.randomUUID()), true, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(96, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("NEW", row.getAsJsonObject("plate").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("sample").get("state").getAsString()); + + BrAPISample sample = new Gson().fromJson(row.getAsJsonObject("sample").getAsJsonObject("brAPIObject"), BrAPISample.class); + BrAPIExternalReference xref = Utilities.getExternalReference(sample.getExternalReferences(), Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.PLATE_SUBMISSIONS)).get(); + + assertFileSaved(validFile, program, UUID.fromString(xref.getReferenceId())); + } + + @Test + @SneakyThrows + public void importObsUnitIdSuccess() { + Program program = createProgram("Import ObsUnitID success", "OBSID", "OBSID", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + + var experimentId = createExperiment(program); + + BrAPITrial trial = brAPITrialDAO.getTrialById(program.getId(), UUID.fromString(experimentId)).get(); + + List ous = ouDAO.getObservationUnitsForTrialDbId(program.getId(), trial.getTrialDbId()); + BrAPIExternalReference obsUnitId = Utilities.getExternalReference(ous.get(0).getExternalReferences(), Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS)).get(); + + List> validFile = new ArrayList<>(); + + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, "A"); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, ""); + validRow.put(Columns.OBS_UNIT_ID, obsUnitId.getReferenceId()); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + + Flowable> call = importTestUtils.uploadDataFile(writeDataToFile(validFile), Map.of(SUBMISSION_NAME, "test-"+UUID.randomUUID()), true, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("NEW", row.getAsJsonObject("plate").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("sample").get("state").getAsString()); + + BrAPISample sample = new Gson().fromJson(row.getAsJsonObject("sample").getAsJsonObject("brAPIObject"), BrAPISample.class); + BrAPIExternalReference xref = Utilities.getExternalReference(sample.getExternalReferences(), Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.PLATE_SUBMISSIONS)).get(); + + assertFileSaved(validFile, program, UUID.fromString(xref.getReferenceId())); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void importMissingGIDAndObsUnitIdFailure(boolean commit) { + Program program = createProgram("Missing GID/ObsUnit ID " + (commit ? "C" : "P"), "MGIOB"+ (commit ? "C" : "P"), "MGIOB"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, null, null); + List> validFile = new ArrayList<>(); + + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A')); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, ""); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + + uploadAndVerifyFailure(program, writeDataToFile(validFile), Columns.GERMPLASM_GID, commit); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void verifyMissingDataThrowsError(boolean commit) { + Program program = createProgram("Missing Req Cols "+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(96), null); + Map base = new HashMap<>(); + base.put(Columns.PLATE_ID, "valid_1"); + base.put(Columns.ROW, "A"); + base.put(Columns.COLUMN, 1); + base.put(Columns.ORGANISM, "TEST"); + base.put(Columns.SPECIES, "TEST"); + base.put(Columns.GERMPLASM_NAME, ""); + base.put(Columns.GERMPLASM_GID, "1"); + base.put(Columns.OBS_UNIT_ID, ""); + base.put(Columns.TISSUE, "TEST"); + base.put(Columns.COMMENTS, "Test sample"); + + createUploadAndVerifyFailure(program, base, Columns.PLATE_ID, commit); + createUploadAndVerifyFailure(program, base, Columns.ROW, commit); + createUploadAndVerifyFailure(program, base, Columns.COLUMN, commit); + createUploadAndVerifyFailure(program, base, Columns.ORGANISM, commit); + createUploadAndVerifyFailure(program, base, Columns.TISSUE, commit); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void importInvalidGIDFailure(boolean commit) { + Program program = createProgram("Invalid GID " + (commit ? "C" : "P"), "INGID"+ (commit ? "C" : "P"), "INGID"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, null, null); + List> validFile = new ArrayList<>(); + + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A')); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, ""); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + + uploadAndVerifyFailure(program, writeDataToFile(validFile), Columns.GERMPLASM_GID, commit); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void importInvalidObsUnitIdFailure(boolean commit) { + Program program = createProgram("Invalid ObsUnit ID " + (commit ? "C" : "P"), "INOBS"+ (commit ? "C" : "P"), "INOBS"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, null, null); + List> validFile = new ArrayList<>(); + + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A')); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, ""); + validRow.put(Columns.OBS_UNIT_ID, "hgfhgfhg"); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + + uploadAndVerifyFailure(program, writeDataToFile(validFile), Columns.OBS_UNIT_ID, commit); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void importConflictingWellsFailure(boolean commit) { + Program program = createProgram("Conflicting Wells " + (commit ? "C" : "P"), "WELL"+ (commit ? "C" : "P"), "WELL"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(2), null); + List> validFile = new ArrayList<>(); + + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A')); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, 1); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A')); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, 2); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + + uploadAndVerifyFailure(program, writeDataToFile(validFile), Columns.ROW + "/" + Columns.COLUMN, commit); + } + + private void assertFileSaved(List> validFile, Program program, UUID submissionId) throws ApiException { + Optional submission = sampleSubmissionService.getSampleSubmission(program, submissionId, true); + assertTrue(submission.isPresent(), "Could not find sample submission by ID: " + submissionId); + for(var row : validFile) { + assertRowSaved(row, program, submission.get()); + } + } + + private Map assertRowSaved(Map expected, Program program, SampleSubmission submission) { + Map ret = new HashMap<>(); + + Optional plate = submission.getPlates().stream().filter(p -> p.getPlateName().equals(expected.get(Columns.PLATE_ID))).findFirst(); + assertTrue(plate.isPresent(), "plate not found"); + + Optional sample = submission.getSamples().stream().filter(s -> s.getPlateName().equals(expected.get(Columns.PLATE_ID)) + && s.getRow().equals(expected.get(Columns.ROW)) + && s.getColumn().equals(expected.get(Columns.COLUMN))) + .findFirst(); + assertTrue(sample.isPresent(), String.format("sample %s%s not found", expected.get(Columns.ROW), expected.get(Columns.COLUMN))); + + assertEquals(expected.get(Columns.ORGANISM), sample.get().getAdditionalInfo().get(BrAPIAdditionalInfoFields.SAMPLE_ORGANISM).getAsString()); + assertEquals(expected.get(Columns.SPECIES), sample.get().getAdditionalInfo().get(BrAPIAdditionalInfoFields.SAMPLE_SPECIES).getAsString()); + assertTrue(sample.get().getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_NAME)); + assertEquals(expected.get(Columns.TISSUE), sample.get().getTissueType()); + assertEquals(expected.get(Columns.COMMENTS), sample.get().getSampleDescription()); + + if(StringUtils.isNotBlank((String)expected.get(Columns.OBS_UNIT_ID))) { + assertTrue(sample.get().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBS_UNIT_ID)); + assertEquals(expected.get(Columns.OBS_UNIT_ID), sample.get().getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBS_UNIT_ID).getAsString()); + } else { + assertEquals(String.valueOf(expected.get(Columns.GERMPLASM_GID)), sample.get().getAdditionalInfo().get(BrAPIAdditionalInfoFields.GID).getAsString()); + } + + return ret; + } + + private Program createProgram(String name, String abbv, String key, String referenceSource, List germplasm, List traits) throws ApiException, DoesNotExistException, ValidatorException, BadRequestException { + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + ProgramRequest programRequest1 = ProgramRequest.builder() + .name(name) + .abbreviation(abbv) + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key(key) + .build(); + + + TestUtils.insertAndFetchTestProgram(gson, client, programRequest1); + + // Get main program + Program program = programService.getByKey(key).get(); + + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId().toString()); + + if(germplasm != null && !germplasm.isEmpty()) { + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", referenceSource)); + newReference.setReferenceID(program.getId().toString()); + + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); + + germplasmDAO.createBrAPIGermplasm(germplasm, program.getId(), null); + } + + if(traits != null && !traits.isEmpty()) { + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + try { + ontologyService.createTraits(program.getId(), traits, user, false); + } catch (ValidatorException e) { + System.err.println(e.getErrors()); + throw e; + } + } + + return program; + } + + private List createGermplasm(int numToCreate) { + List germplasm = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String gid = ""+(i+1); + BrAPIGermplasm testGermplasm = new BrAPIGermplasm(); + testGermplasm.setGermplasmName(String.format("Germplasm %s [TEST-%s]", gid, gid)); + testGermplasm.setSeedSource("Wild"); + testGermplasm.setAccessionNumber(gid); + testGermplasm.setDefaultDisplayName(String.format("Germplasm %s", gid)); + JsonObject additionalInfo = new JsonObject(); + additionalInfo.addProperty("importEntryNumber", gid); + additionalInfo.addProperty("breedingMethod", "Allopolyploid"); + testGermplasm.setAdditionalInfo(additionalInfo); + List externalRef = new ArrayList<>(); + BrAPIExternalReference testReference = new BrAPIExternalReference(); + testReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); + testReference.setReferenceID(UUID.randomUUID().toString()); + externalRef.add(testReference); + testGermplasm.setExternalReferences(externalRef); + germplasm.add(testGermplasm); + } + + return germplasm; + } + + private void createUploadAndVerifyFailure(Program program, Map base, String columnToRemove, boolean commit) throws IOException, InterruptedException { + Map invalidRow = new HashMap<>(base); + invalidRow.remove(columnToRemove); + uploadAndVerifyFailure(program, writeDataToFile(List.of(invalidRow)), columnToRemove, commit); + } + + private JsonObject uploadAndVerifyFailure(Program program, File file, String expectedColumnError, boolean commit) throws InterruptedException, IOException { + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(SUBMISSION_NAME, "test-"+UUID.randomUUID()), true, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); + assertEquals(1, rowErrors.size()); + JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); + assertEquals(1, fieldErrors.size()); + JsonObject error = fieldErrors.get(0).getAsJsonObject(); + assertEquals(expectedColumnError, error.get("field").getAsString()); + assertEquals(422, error.get("httpStatusCode").getAsInt()); + + return result; + } + + public File writeDataToFile(List> data) throws IOException { + File file = File.createTempFile("test", ".csv"); + + List columns = new ArrayList<>(); + columns.add(Column.builder().value(Columns.PLATE_ID).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.ROW).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.COLUMN).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.ORGANISM).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.SPECIES).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.GERMPLASM_NAME).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.GERMPLASM_GID).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.OBS_UNIT_ID).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.TISSUE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.COMMENTS).dataType(Column.ColumnDataType.STRING).build()); + + ByteArrayOutputStream byteArrayOutputStream = CSVWriter.writeToCSV(columns, data); + FileOutputStream fos = new FileOutputStream(file); + fos.write(byteArrayOutputStream.toByteArray()); + + return file; + } + + private String createExperiment(Program program) throws IOException, InterruptedException { + Flowable> call = client.exchange( + GET("/import/mappings?importName=ExperimentsTemplateMap").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + String expMappingId = JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + JsonObject importResult = importTestUtils.uploadAndFetch( + importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null), + null, + true, + client, + program, + expMappingId); + return importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + } + + private Map makeExpImportRow(String environment) { + Map row = new HashMap<>(); + row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); + row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); + row.put(ExperimentObservation.Columns.EXP_TITLE, "Test Exp"); + row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); + row.put(ExperimentObservation.Columns.ENV, environment); + row.put(ExperimentObservation.Columns.ENV_LOCATION, "Location A"); + row.put(ExperimentObservation.Columns.ENV_YEAR, "2023"); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, "a-1"); + row.put(ExperimentObservation.Columns.REP_NUM, "1"); + row.put(ExperimentObservation.Columns.BLOCK_NUM, "1"); + row.put(ExperimentObservation.Columns.ROW, "1"); + row.put(ExperimentObservation.Columns.COLUMN, "1"); + return row; + } + +}