diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 6ea30cd20..611a5e5fb 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -18,7 +18,6 @@ package org.breedinginsight.brapi.v2.constants; public final class BrAPIAdditionalInfoFields { - public static final String GERMPLASM_LIST_ENTRY_NUMBERS = "listEntryNumbers"; public static final String GERMPLASM_LIST_ID = "listId"; public static final String GERMPLASM_RAW_PEDIGREE = "rawPedigree"; public static final String GERMPLASM_PEDIGREE_BY_NAME = "pedigreeByName"; diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java index eafde7e1e..4824993b6 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapi.v2.services; +import com.google.gson.Gson; import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; import io.micronaut.http.server.types.files.StreamedFile; @@ -24,6 +25,7 @@ import org.breedinginsight.services.writers.ExcelWriter; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; @@ -31,7 +33,6 @@ import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.*; -import java.util.function.ToIntFunction; import java.util.stream.Collectors; @Slf4j @@ -44,6 +45,7 @@ public class BrAPIGermplasmService { private final BrAPIGermplasmDAO germplasmDAO; private final ProgramService programService; private final BrAPIListDAO brAPIListDAO; + private final Gson gson = new Gson(); @Inject public BrAPIGermplasmService(BrAPIListDAO brAPIListDAO, ProgramService programService, BrAPIGermplasmDAO germplasmDAO) { @@ -80,10 +82,36 @@ public Optional getGermplasmByDBID(UUID programId, String germpl return germplasmDAO.getGermplasmByDBID(germplasmId, programId); } - public List> processListData(List germplasm, UUID germplasmListId){ + public List> processListData(List germplasm, BrAPIListDetails germplasmList){ + Map germplasmByName = new HashMap<>(); + for (BrAPIGermplasm g: germplasm) { + germplasmByName.put(g.getGermplasmName(), g); + } + List> processedData = new ArrayList<>(); - for (BrAPIGermplasm germplasmEntry: germplasm) { + // This holds the BrAPI list items or all germplasm in a program if the list is null. + List orderedGermplasmNames = new ArrayList<>(); + if (germplasmList == null) { + orderedGermplasmNames = germplasm.stream().sorted((left, right) -> { + Integer leftAccessionNumber = Integer.parseInt(left.getAccessionNumber()); + Integer rightAccessionNumber = Integer.parseInt(right.getAccessionNumber()); + return leftAccessionNumber.compareTo(rightAccessionNumber); + }).map(BrAPIGermplasm::getGermplasmName).collect(Collectors.toList()); + } else { + orderedGermplasmNames = germplasmList.getData(); + } + + // For export, assign entry number sequentially based on BrAPI list order. + int entryNumber = 0; + for (String germplasmName: orderedGermplasmNames) { + // Increment entryNumber. + ++entryNumber; + // Strip program key and accession number from germplasm name. + germplasmName = Utilities.removeUnknownProgramKey(germplasmName); // TODO: could move to the germplasmList != null codepath. + // Lookup the BrAPI germplasm in the map. + BrAPIGermplasm germplasmEntry = germplasmByName.get(germplasmName); + HashMap row = new HashMap<>(); row.put("GID", Integer.valueOf(germplasmEntry.getAccessionNumber())); row.put("Germplasm Name", germplasmEntry.getGermplasmName()); @@ -92,12 +120,11 @@ public List> processListData(List germplasm, row.put("Source", source); // Use the entry number in the list map if generated - if(new UUID(0,0).compareTo(germplasmListId) == 0) { + if(germplasmList == null) { // Not downloading a real list, use GID (https://breedinginsight.atlassian.net/browse/BI-2266). row.put("Entry No", Integer.valueOf(germplasmEntry.getAccessionNumber())); } else { - row.put("Entry No", germplasmEntry.getAdditionalInfo() - .getAsJsonObject(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ENTRY_NUMBERS).get(germplasmListId.toString()).getAsInt()); + row.put("Entry No", entryNumber); } //If germplasm was imported with an external UID, it will be stored in external reference with same source as seed source @@ -156,13 +183,40 @@ public List getGermplasmByList(UUID programId, String listDbId) // get list BrAPI germplasm variables List germplasmNames = listResponse.getResult().getData(); List germplasm = germplasmDAO.getGermplasmByRawName(germplasmNames, programId); + Map germplasmByName = new HashMap<>(); - // set the list ID in the germplasm additional info - germplasm.forEach(g -> g.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ID, listId)); - return germplasm; + for (BrAPIGermplasm g : germplasm) { + // set the list ID in the germplasm additional info + germplasm.forEach(x -> x.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ID, listId)); + // Add to map. + germplasmByName.put(g.getGermplasmName(), g); + } + + // Get the program key. + String programKey = programService.getById(programId) + .orElseThrow(ApiException::new) + .getKey(); + + // Build list from BrAPI list that preserves ordering and duplicates and assigns sequential entry numbers. + List germplasmList = new ArrayList<>(); + int entryNumber = 0; + for (String germplasmName : germplasmNames) { + ++entryNumber; + BrAPIGermplasm listEntry = cloneBrAPIGermplasm(germplasmByName.get(Utilities.removeProgramKeyAndUnknownAdditionalData(germplasmName, programKey))); + // Set entry number. + listEntry.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_IMPORT_ENTRY_NUMBER, entryNumber); + germplasmList.add(listEntry); + } + + return germplasmList; } else throw new ApiException(); } + private BrAPIGermplasm cloneBrAPIGermplasm(BrAPIGermplasm germplasm) { + // Serialize then deserialize to deep copy. + return (BrAPIGermplasm) gson.fromJson(gson.toJson(germplasm), BrAPIGermplasm.class); + } + public DownloadFile exportGermplasm(UUID programId, FileType fileExtension) throws IllegalArgumentException, ApiException, IOException { List columns = GermplasmFileColumns.getOrderedColumns(); @@ -184,7 +238,7 @@ public DownloadFile exportGermplasm(UUID programId, FileType fileExtension) thro StreamedFile downloadFile; //Convert list data to List> data to pass into file writer - List> processedData = processListData(germplasm, new UUID(0,0)); + List> processedData = processListData(germplasm, null); if (fileExtension == FileType.CSV){ downloadFile = CSVWriter.writeToDownload(columns, processedData, fileExtension); @@ -205,10 +259,6 @@ public DownloadFile exportGermplasmList(UUID programId, String listId, FileType List germplasmNames = listData.getData(); List germplasm = germplasmDAO.getGermplasmByRawName(germplasmNames, programId); - //processGermplasmForDisplay, numbers - UUID germplasmListId = getGermplasmListId(listData); - germplasm.sort(Comparator.comparingInt(getEntryNumber(germplasmListId))); - String listName = listData.getListName(); Optional optionalProgram = programService.getById(programId); if (optionalProgram.isPresent()) { @@ -218,7 +268,7 @@ public DownloadFile exportGermplasmList(UUID programId, String listId, FileType String fileName = createFileName(listData, listName); StreamedFile downloadFile; //Convert list data to List> data to pass into file writer - List> processedData = processListData(germplasm, germplasmListId); + List> processedData = processListData(germplasm, listData); if (fileExtension == FileType.CSV){ downloadFile = CSVWriter.writeToDownload(columns, processedData, fileExtension); @@ -254,32 +304,6 @@ private boolean hasListExternalReference(List refs) thro return refs.stream().anyMatch(e -> referenceSource.concat("/lists").equals(e.getReferenceSource())); } - private ToIntFunction getEntryNumber(UUID germplasmListId) throws IllegalArgumentException { - if(germplasmListId.compareTo(new UUID(0,0)) == 0) { - return this::getImportEntryNumber; - } else { - return g -> getGermplasmListEntryNumber(g, germplasmListId); - } - } - - private Integer getImportEntryNumber(BrAPIGermplasm g) throws IllegalArgumentException { - if(Objects.nonNull(g.getAdditionalInfo()) && - g.getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_IMPORT_ENTRY_NUMBER)) { - return g.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_IMPORT_ENTRY_NUMBER).getAsInt(); - } else { - throw new IllegalArgumentException(); - } - } - private Integer getGermplasmListEntryNumber(BrAPIGermplasm g, UUID germplasmListId) throws IllegalArgumentException { - if(Objects.nonNull(g.getAdditionalInfo()) && - g.getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ENTRY_NUMBERS)) { - return g.getAdditionalInfo().getAsJsonObject(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ENTRY_NUMBERS) - .get(germplasmListId.toString()).getAsInt(); - } else { - throw new IllegalArgumentException(); - } - } - private String createFileName(BrAPIListDetails listData, String listName) { //TODO change timestamp to edit date when editing functionality is added String timestamp; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java index f1a8dd1b0..00d89e4db 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java @@ -185,13 +185,6 @@ public void updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID } } - // Add germplasm to the new list - JsonObject listEntryNumbers = germplasm.getAdditionalInfo().getAsJsonObject(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ENTRY_NUMBERS); - if(listEntryNumbers == null) { - listEntryNumbers = new JsonObject(); - germplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ENTRY_NUMBERS, listEntryNumbers); - } - listEntryNumbers.addProperty(listId.toString(), entryNo); germplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_IMPORT_ENTRY_NUMBER, entryNo); //so the preview UI shows correctly // TODO: figure out why clear this out: brapi-server @@ -241,9 +234,6 @@ public BrAPIGermplasm constructBrAPIGermplasm(ProgramBreedingMethodEntity breedi createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_ID, user.getId().toString()); createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_NAME, user.getName()); germplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_BY, createdBy); - Map listEntryNumbers = new HashMap<>(); - listEntryNumbers.put(listId, entryNo); - germplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ENTRY_NUMBERS, listEntryNumbers); //TODO: Need to check that the acquisition date it in date format //brAPIGermplasm.setAcquisitionDate(pedigreeImport.getGermplasm().getAcquisitionDate()); germplasm.setCountryOfOriginCode(getCountryOfOrigin()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index eb1a1c06b..25555b28e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -16,6 +16,7 @@ */ package org.breedinginsight.brapps.importer.services.processors; +import com.google.gson.Gson; import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.HttpStatus; @@ -70,6 +71,7 @@ public class GermplasmProcessor implements Processor { private final BrAPIListDAO brAPIListDAO; private final DSLContext dsl; private final BrAPIGermplasmDAO brAPIGermplasmDAO; + private final Gson gson = new Gson(); Map> germplasmByAccessionNumber = new HashMap<>(); Map fileGermplasmByName = new HashMap<>(); @@ -263,6 +265,16 @@ public Map process(ImportUpload upload, List
entryNumberCounts = new HashMap<>(); List userProvidedEntryNumbers = new ArrayList<>(); ValidationErrors validationErrors = new ValidationErrors(); + // Sort importRows by entry number (if present). + importRows.sort((left, right) -> { + if (left.getGermplasm().getEntryNo() == null || right.getGermplasm().getEntryNo() == null) { + return 0; + } else { + Integer leftEntryNo = Integer.parseInt(left.getGermplasm().getEntryNo()); + Integer rightEntryNo = Integer.parseInt(right.getGermplasm().getEntryNo()); + return leftEntryNo.compareTo(rightEntryNo); + } + }); for (int i = 0; i < importRows.size(); i++) { log.debug("processing germplasm row: " + (i+1)); BrAPIImport brapiImport = importRows.get(i); @@ -373,6 +385,8 @@ private boolean processExistingGermplasm(Germplasm germplasm, ValidationErrors v String gid = germplasm.getAccessionNumber(); if (germplasmByAccessionNumber.containsKey(gid)) { existingGermplasm = germplasmByAccessionNumber.get(gid).getBrAPIObject(); + // Serialize and deserialize to deep copy + existingGermplasm = gson.fromJson(gson.toJson(existingGermplasm), BrAPIGermplasm.class); } else { //should be caught in getExistingBrapiData ValidationError ve = new ValidationError("GID", String.format(missingGID, gid), HttpStatus.NOT_FOUND); diff --git a/src/main/java/org/breedinginsight/utilities/response/mappers/GermplasmQueryMapper.java b/src/main/java/org/breedinginsight/utilities/response/mappers/GermplasmQueryMapper.java index cf7b3c227..e75c7c8aa 100644 --- a/src/main/java/org/breedinginsight/utilities/response/mappers/GermplasmQueryMapper.java +++ b/src/main/java/org/breedinginsight/utilities/response/mappers/GermplasmQueryMapper.java @@ -31,21 +31,13 @@ public class GermplasmQueryMapper extends AbstractQueryMapper { public GermplasmQueryMapper() { fields = Map.ofEntries( - Map.entry("importEntryNumber", (germplasm) ->{ + Map.entry("importEntryNumber", (germplasm) -> { String entryNumber = null; if (germplasm.getAdditionalInfo() != null) { // if additionalInfo contains the importEntryNumber key then return the value if (germplasm.getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_IMPORT_ENTRY_NUMBER)) { entryNumber = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_IMPORT_ENTRY_NUMBER).getAsString(); } - - // if additionalInfo has both listEntryNumbers and listId keys then return the entry number - // mapped to the listId - if (germplasm.getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ENTRY_NUMBERS) - && germplasm.getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ID)) { - String listId = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ID).getAsString(); - entryNumber = germplasm.getAdditionalInfo().getAsJsonObject(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ENTRY_NUMBERS).get(listId).getAsString(); - } } return entryNumber; }), diff --git a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java index d75f524ca..9a58281bf 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java @@ -91,28 +91,6 @@ public void setup() { testUser = (BiUserEntity) setupObjects.get("testUser"); } - // Tests - // Minimum import required fields, success - // Female parent not exist db id, throw error - // Female parent not exist entry number, throw error - // Male parent not exist db id, throw error - // Female parent not exist entry number, throw error - // Bad breeding method, throw error - // Some entry numbers, throw an error - // Numerical entry numbers - // Required fields missing, throw error - // Missing required headers - // Missing optional headers - // No entry numbers, automatic generation - // Full import, success - // Preview, non-preview fields not shown - // Male db, no female db id, pedigree string is null - // Circular parent dependency - // Name and description success - // Name only success - // Dup list name test - // Missing required fields tests - @Test @SneakyThrows @Order(1) @@ -133,8 +111,6 @@ public void minimalImportUserSpecifiedEntryNumbersSuccess() { for (int i = 0; i < previewRows.size(); i++) { JsonObject germplasm = previewRows.get(i).getAsJsonObject().getAsJsonObject("germplasm").getAsJsonObject("brAPIObject"); germplasmNames.add(germplasm.get("germplasmName").getAsString()); - int finalI = i; - gson.fromJson(germplasm.getAsJsonObject("additionalInfo").getAsJsonObject("listEntryNumbers"), Map.class).forEach((listId, entryNumber) -> assertEquals(Integer.toString(finalI +1), entryNumber, "Wrong entry number")); } // Check the germplasm list @@ -145,18 +121,13 @@ public void minimalImportUserSpecifiedEntryNumbersSuccess() { @SneakyThrows @Order(2) public void fullImportPreviewSuccess() { - File file = new File("src/test/resources/files/germplasm_import/full_import.csv"); - Table fileData = Table.read().file("src/test/resources/files/germplasm_import/full_import.csv"); + + String pathname = "src/test/resources/files/germplasm_import/full_import.csv"; + Table fileData = Table.read().file(pathname); String listName = "FullList"; String listDescription = "A full import"; - Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, listName, GERM_LIST_DESC, listDescription), false, client, validProgram, germplasmMappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + JsonObject result = importGermplasm(pathname, listName, listDescription, false); assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); @@ -205,18 +176,13 @@ public void fullImportPreviewSuccess() { @SneakyThrows @Order(3) public void fullImportCommitSuccess() { - File file = new File("src/test/resources/files/germplasm_import/full_import.csv"); - Table fileData = Table.read().file("src/test/resources/files/germplasm_import/full_import.csv"); + String pathname = "src/test/resources/files/germplasm_import/full_import.csv"; + Table fileData = Table.read().file(pathname); String listName = "FullList"; String listDescription = "A full import"; - Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, listName, GERM_LIST_DESC, listDescription), true, client, validProgram, germplasmMappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + JsonObject result = importGermplasm(pathname, listName, listDescription, true); assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); @@ -306,6 +272,130 @@ public void duplicateNameMarksDuplicates() { } } + @Test + @SneakyThrows + @Order(5) + public void duplicateListEntriesSuccess() { + // Test that a germplasm list import referencing existing germplasm by GID can have duplicate entries. + + String pathname = "src/test/resources/files/germplasm_import/valid_duplicate_entries.csv"; + Table fileData = Table.read().file(pathname); + String listName = "Duplicates"; + String listDescription = "New list referencing existing germplasm by GID with duplicate entries"; + + JsonObject result = importGermplasm(pathname, listName, listDescription, true); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + List germplasmNames = new ArrayList<>(); + for (int i = 0; i < previewRows.size(); i++) { + JsonObject germplasm = previewRows.get(i).getAsJsonObject().getAsJsonObject("germplasm").getAsJsonObject("brAPIObject"); + germplasmNames.add(germplasm.get("germplasmName").getAsString()); + checkBasicResponse(germplasm, fileData, i); + + // Germplasm name (display name) + String expectedGermplasmName = String.format("%s [%s-%s]", fileData.getString(i, "Germplasm Name"), validProgram.getKey(), germplasm.get("accessionNumber").getAsString()); + assertEquals(expectedGermplasmName, germplasm.get("germplasmName").getAsString()); + // Created Date + JsonObject additionalInfo = germplasm.getAsJsonObject("additionalInfo"); + assertTrue(additionalInfo.has(BrAPIAdditionalInfoFields.CREATED_DATE), "createdDate is missing"); + // Accession Number + assertTrue(germplasm.has("accessionNumber"), "accessionNumber missing"); + // Pedigree (germplasm names) + String pedigree = germplasm.get("pedigree").getAsString(); + String mother = !pedigree.isBlank() ? pedigree.split("/")[0] : null; + String father = !pedigree.isBlank() && pedigree.split("/").length > 1 ? pedigree.split("/")[1] : null; + String regexMatcher = "^(.*\\b) \\[([A-Z]{2,6})-(\\d+)\\]$"; + + // External Reference germplasm + JsonArray externalReferences = germplasm.getAsJsonArray("externalReferences"); + boolean referenceFound = false; + for (JsonElement reference: externalReferences) { + String referenceSource = reference.getAsJsonObject().get("referenceSource").getAsString(); + if (referenceSource.equals(BRAPI_REFERENCE_SOURCE)) { + referenceFound = true; + break; + } + } + assertTrue(referenceFound, "Germplasm UUID reference not found"); + + // Synonyms + String[] splitGermplasmName = germplasm.get("germplasmName").getAsString().split(" "); + String scope = splitGermplasmName[splitGermplasmName.length - 1]; + JsonArray synonyms = germplasm.getAsJsonArray("synonyms"); + for (JsonElement synonym: synonyms) { + String synonymName = synonym.getAsJsonObject().get("synonym").getAsString(); + assertNotNull(synonymName); + assertTrue(synonymName.contains(scope), "Germplasm synonym was not properly scoped"); + } + } + + // Check the germplasm list + checkGermplasmList(Germplasm.constructGermplasmListName(listName, validProgram), listDescription, germplasmNames); + } + + @Test + @SneakyThrows + @Order(6) + public void newAndExistingListEntriesSuccess() { + // Test that a germplasm list import that both creates new germplasm (GID empty) and references existing germplasm (by GID) succeeds. + + String pathname = "src/test/resources/files/germplasm_import/new_and_existing_list_entries.csv"; + Table fileData = Table.read().file(pathname); + String listName = "New and Existing Germplasm"; + String listDescription = "New list containing both new and existing germplasm"; + + JsonObject result = importGermplasm(pathname, listName, listDescription, true); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + List germplasmNames = new ArrayList<>(); + for (int i = 0; i < previewRows.size(); i++) { + JsonObject germplasm = previewRows.get(i).getAsJsonObject().getAsJsonObject("germplasm").getAsJsonObject("brAPIObject"); + germplasmNames.add(germplasm.get("germplasmName").getAsString()); + checkBasicResponse(germplasm, fileData, i); + + // Germplasm name (display name) + String expectedGermplasmName = String.format("%s [%s-%s]", fileData.getString(i, "Germplasm Name"), validProgram.getKey(), germplasm.get("accessionNumber").getAsString()); + assertEquals(expectedGermplasmName, germplasm.get("germplasmName").getAsString()); + // Created Date + JsonObject additionalInfo = germplasm.getAsJsonObject("additionalInfo"); + assertTrue(additionalInfo.has(BrAPIAdditionalInfoFields.CREATED_DATE), "createdDate is missing"); + // Accession Number + assertTrue(germplasm.has("accessionNumber"), "accessionNumber missing"); + // Pedigree (germplasm names) + String pedigree = germplasm.get("pedigree").getAsString(); + String mother = !pedigree.isBlank() ? pedigree.split("/")[0] : null; + String father = !pedigree.isBlank() && pedigree.split("/").length > 1 ? pedigree.split("/")[1] : null; + String regexMatcher = "^(.*\\b) \\[([A-Z]{2,6})-(\\d+)\\]$"; + + // External Reference germplasm + JsonArray externalReferences = germplasm.getAsJsonArray("externalReferences"); + boolean referenceFound = false; + for (JsonElement reference: externalReferences) { + String referenceSource = reference.getAsJsonObject().get("referenceSource").getAsString(); + if (referenceSource.equals(BRAPI_REFERENCE_SOURCE)) { + referenceFound = true; + break; + } + } + assertTrue(referenceFound, "Germplasm UUID reference not found"); + + // Synonyms + String[] splitGermplasmName = germplasm.get("germplasmName").getAsString().split(" "); + String scope = splitGermplasmName[splitGermplasmName.length - 1]; + JsonArray synonyms = germplasm.getAsJsonArray("synonyms"); + for (JsonElement synonym: synonyms) { + String synonymName = synonym.getAsJsonObject().get("synonym").getAsString(); + assertNotNull(synonymName); + assertTrue(synonymName.contains(scope), "Germplasm synonym was not properly scoped"); + } + } + + // Check the germplasm list + checkGermplasmList(Germplasm.constructGermplasmListName(listName, validProgram), listDescription, germplasmNames); + } + @Test @SneakyThrows public void OnlyMaleParentPreviewSuccess() { @@ -563,8 +653,6 @@ public void headerCaseInsensitive() { for (int i = 0; i < previewRows.size(); i++) { JsonObject germplasm = previewRows.get(i).getAsJsonObject().getAsJsonObject("germplasm").getAsJsonObject("brAPIObject"); germplasmNames.add(germplasm.get("germplasmName").getAsString()); - int finalI = i; - gson.fromJson(germplasm.getAsJsonObject("additionalInfo").getAsJsonObject("listEntryNumbers"), Map.class).forEach((listId, entryNumber) -> assertEquals(Integer.toString(finalI +1), entryNumber, "Wrong entry number")); } // Check the germplasm list @@ -645,13 +733,22 @@ public void selfReferenceParentError() { assertEquals(GermplasmProcessor.circularDependency, result.getAsJsonObject("progress").get("message").getAsString()); } + private JsonObject importGermplasm(String pathname, String listName, String listDescription, Boolean commit) throws InterruptedException { + File file = new File(pathname); + + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, listName, GERM_LIST_DESC, listDescription), commit, client, validProgram, germplasmMappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); + return JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + } public void checkBasicResponse(JsonObject germplasm, Table fileData, Integer i) { // Germplasm display name assertEquals(fileData.getString(i, "Germplasm Name"), germplasm.get("defaultDisplayName").getAsString(), "Wrong display name"); - // Entry Number - gson.fromJson(germplasm.getAsJsonObject("additionalInfo").getAsJsonObject("listEntryNumbers"), Map.class).forEach((listId, entryNumber) -> assertEquals(fileData.getString(i, "Entry No"), entryNumber, "Wrong entry number")); JsonObject additionalInfo = germplasm.getAsJsonObject("additionalInfo"); // Created By User ID assertEquals(testUser.getId().toString(), additionalInfo.getAsJsonObject(BrAPIAdditionalInfoFields.CREATED_BY).get(BrAPIAdditionalInfoFields.CREATED_BY_USER_ID).getAsString(), "Wrong createdBy userId"); @@ -726,6 +823,8 @@ public void checkGermplasmList(String listName, String listDescription, List found = new ArrayList<>(); + // Check that the list size is correct. + assertEquals(germplasmNames.size(), germplasmList.size()); for (String germplasmName: germplasmNames) { for (JsonElement listElement: germplasmList) { if (listElement.getAsString().equals(germplasmName)) { diff --git a/src/test/resources/files/germplasm_import/minimal_germplasm_import.csv b/src/test/resources/files/germplasm_import/minimal_germplasm_import.csv index 7ff102940..c45873707 100644 --- a/src/test/resources/files/germplasm_import/minimal_germplasm_import.csv +++ b/src/test/resources/files/germplasm_import/minimal_germplasm_import.csv @@ -1,4 +1,4 @@ GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -,Germplasm 1,BCR,Wild,,,,,,, -,Germplasm 2,BCR,Cultivated,,,,,,, -,Germplasm 3,BCR,Kinda Wild,,,,,,, \ No newline at end of file +,Germplasm 1,BCR,Wild,,,,,,1414, +,Germplasm 2,BCR,Cultivated,,,,,,1515, +,Germplasm 3,BCR,Kinda Wild,,,,,,1616, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/new_and_existing_list_entries.csv b/src/test/resources/files/germplasm_import/new_and_existing_list_entries.csv new file mode 100644 index 000000000..8083f3b86 --- /dev/null +++ b/src/test/resources/files/germplasm_import/new_and_existing_list_entries.csv @@ -0,0 +1,6 @@ +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +1,Germplasm 1,BCR,Wild,,,1,,,1414, +2,Germplasm 2,BCR,Cultivated,,,2,,,1515, +3,Germplasm 3,BCR,Kinda Wild,,,3,,,1616, +,New Germ 1,BCR,Wild,1,2,4,,,3142,test01 +,New Germ 2,BCR,Wild,1,2,5,,,4211,test02 \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/valid_duplicate_entries.csv b/src/test/resources/files/germplasm_import/valid_duplicate_entries.csv new file mode 100644 index 000000000..f6238ace9 --- /dev/null +++ b/src/test/resources/files/germplasm_import/valid_duplicate_entries.csv @@ -0,0 +1,5 @@ +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +1,Germplasm 1,BCR,Wild,,,4,,,1414, +2,Germplasm 2,BCR,Cultivated,,,3,,,1515, +2,Germplasm 2,BCR,Cultivated,,,2,,,1515, +1,Germplasm 1,BCR,Wild,,,1,,,1414, \ No newline at end of file