From da3bff2c4bb29f81aa3a24c5499c324e339b4c50 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Wed, 14 Sep 2022 14:20:14 -0400 Subject: [PATCH 01/23] [BI-1473] - Allow user to append existing germplasm with optional details --- .../brapps/importer/model/base/Germplasm.java | 5 +++++ .../V0.7.2__germplasm_import_add_GID.sql | 21 +++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 src/main/resources/db/migration/V0.7.2__germplasm_import_add_GID.sql diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java index c8c323619..fe85bf1f0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java @@ -107,6 +107,10 @@ public class Germplasm implements BrAPIObject { @ImportFieldMetadata(id="collection", name="Family Name", description = "The name of the family this germplasm is a part of.") private String collection; + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="germplasmAccessionNumber", name="Accession Number", description = "The accession number of the germplasm if germplasm is being re-imported with updated synonyms/parents.") + private String accessionNumber; + // Removed for now, need to add to breedbase /*@ImportType(type=ImportFieldType.LIST, clazz=GermplasmAttribute.class) private List germplasmAttributes;*/ @@ -149,6 +153,7 @@ public BrAPIGermplasm constructBrAPIGermplasm(BreedingMethodEntity breedingMetho germplasm.setDefaultDisplayName(getGermplasmName()); germplasm.setGermplasmPUI(getGermplasmPUI()); germplasm.setCollection(getCollection()); + germplasm.setGermplasmDbId(getAccessionNumber()); germplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_IMPORT_ENTRY_NUMBER, entryNo); germplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_GID, getFemaleParentDBID()); germplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID, getMaleParentDBID()); diff --git a/src/main/resources/db/migration/V0.7.2__germplasm_import_add_GID.sql b/src/main/resources/db/migration/V0.7.2__germplasm_import_add_GID.sql new file mode 100644 index 000000000..0739723ef --- /dev/null +++ b/src/main/resources/db/migration/V0.7.2__germplasm_import_add_GID.sql @@ -0,0 +1,21 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +update importer_mapping +set mapping = '[{"id": "f8d43c7e-a618-4c16-8829-3085f7202a67", "mapping": [{"id": "f384837e-ad8d-4dbe-b54e-87b57070bed1", "value": {"fileFieldName": "Name"}, "objectId": "germplasmName"}, {"id": "39628d14-458b-429b-8e66-bb48e0445a83", "value": {"fileFieldName": "Breeding Method"}, "objectId": "breedingMethod"}, {"id": "f1ba63e1-f5e4-433f-a53e-1c2f3e2fa71f", "value": {"fileFieldName": "Source"}, "objectId": "germplasmSource"}, {"id": "f5892565-f888-4596-be82-ab8eeabf37ce", "value": {"fileFieldName": "External UID"}, "objectId": "externalUID"}, {"id": "65507e5d-2d66-4595-8763-e772fe25c870", "value": {"fileFieldName": "Entry No"}, "objectId": "entryNo"}, {"id": "3eae24c1-ca4a-48a2-96d0-3cf4630acd3a", "value": {"fileFieldName": "Female Parent GID"}, "objectId": "femaleParentDBID"}, {"id": "2dbd7262-93a1-44b0-86b7-f5fca290b965", "value": {"fileFieldName": "Male Parent GID"}, "objectId": "maleParentDBID"}, {"id": "6f7f1539-6e8f-4ede-b7d3-3423cc63abec", "value": {"fileFieldName": "Female Parent Entry No"}, "objectId": "femaleParentEntryNo"}, {"id": "25fe9954-bca7-42f1-818a-5f71e242fa1f", "value": {"fileFieldName": "Male Parent Entry No"}, "objectId": "maleParentEntryNo"}, {"id": "b910adfe-a474-47a0-8410-514578898436", "value": {"fileFieldName": "Synonyms"}, "objectId": "synonyms"}, {"id": "15836d5f-8194-40a8-a771-114eaae31eb4", "objectId": "germplasmPUI"}, {"id": "675b6af8-5a17-4146-a503-2e4e1a65d5fa", "objectId": "acquisitionDate"}, {"id": "69a3bd3c-cebc-435c-acdd-0be62dda25ed", "objectId": "countryOfOrigin"}, {"id": "8ab25267-20f2-450e-89ca-21634ff8fadb", "objectId": "collection"}, {"id": "bc09c6e1-866f-45c3-a285-a25859e8c982", "value": {"fileFieldName": "GID"}, "objectId": "germplasmAccessionNumber"}, {"id": "ce1701e2-2f61-4250-8595-9536e3f5ddcf", "objectId": "AdditionalInfo"}, {"id": "3470e9df-a028-45b7-943f-198bc62b6dbe", "objectId": "ExternalReference"}], "objectId": "Germplasm"}]', + file = '[{"Name": "BITest Pinot Noir", "Source": "Unknown", "Entry No": "1", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": "", "Synonyms": "test1;test2", "GID": ""}, {"Name": "BITest Pixie", "Source": "Winters Nursery", "Entry No": "2", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": "1", "Synonyms": "test1;test2", "GID": ""}, {"Name": "BITest BI002", "Source": "Ithaca Nursery", "Entry No": "7", "External UID": "12231321", "Breeding Method": "Biparental cross", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": "2", "Synonyms": "test1;test2", "GID": ""}, {"Name": "BITest BI003", "Source": "Ithaca Nursery", "Entry No": "8", "External UID": "", "Breeding Method": "BPC", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "7", "Female Parent Entry No": "2", "Synonyms": "test1;test2", "GID": ""}, {"Name": "BITest Pinot Noir", "Source": "Unknown", "Entry No": "3", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": "", "Synonyms": "test1;test2", "GID": ""}, {"Name": "BITest Pixie", "Source": "Winters Nursery", "Entry No": "4", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "640e8b58-1b1c-44a6-91a6-85b2b773376b", "Female Parent GID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": "", "Synonyms": "test1;test2", "GID": ""}, {"Name": "BITest BI002", "Source": "Ithaca Nursery", "Entry No": "5", "External UID": "12231321", "Breeding Method": "Biparental cross", "Male Parent GID": "", "Female Parent GID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": "2", "Synonyms": "test1;test2", "GID": ""}]' +where name = 'GermplasmTemplateMap'; \ No newline at end of file From d704498a3c27e4bacc2b1284e0e0a77950174736 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Wed, 14 Sep 2022 15:58:25 -0400 Subject: [PATCH 02/23] Update download link and test files --- .../germplasm_import/bad_breeding_methods.csv | 8 ++++---- .../circular_parent_dependencies.csv | 8 ++++---- .../files/germplasm_import/duplicate_db_names.csv | 14 +++++++------- .../germplasm_import/duplicate_entry_numbers.csv | 10 +++++----- .../germplasm_import/empty_required_fields.csv | 8 ++++---- .../empty_values_required_fields.csv | 10 +++++----- .../germplasm_import/female_dbid_not_exist.csv | 8 ++++---- .../female_entry_number_not_exist.csv | 8 ++++---- .../files/germplasm_import/full_import.csv | 8 ++++---- .../germplasm_import/germplasm_column_casing.csv | 4 ++-- .../male_dbid_no_female_success.csv | 8 ++++---- .../files/germplasm_import/male_dbid_not_exist.csv | 8 ++++---- .../male_entry_number_not_exist.csv | 8 ++++---- .../germplasm_import/minimal_germplasm_import.csv | 8 ++++---- .../germplasm_import/missing_optional_header.csv | 8 ++++---- .../germplasm_import/missing_required_header.csv | 8 ++++---- .../no_female_parent_blank_pedigree.csv | 8 ++++---- .../nonnumerical_entry_numbers.csv | 10 +++++----- .../self_ref_parent_dependencies.csv | 8 ++++---- .../files/germplasm_import/some_entry_numbers.csv | 10 +++++----- 20 files changed, 85 insertions(+), 85 deletions(-) diff --git a/src/test/resources/files/germplasm_import/bad_breeding_methods.csv b/src/test/resources/files/germplasm_import/bad_breeding_methods.csv index eabba0bbc..9faa60aee 100644 --- a/src/test/resources/files/germplasm_import/bad_breeding_methods.csv +++ b/src/test/resources/files/germplasm_import/bad_breeding_methods.csv @@ -1,4 +1,4 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germplasm 1,ANE,Wild,,,,,,, -Germplasm 2,BAD,Wild,,,,,,, -Germplasm 3,BAD1,Wild,,,,,,, \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,ANE,Wild,,,,,,, +,Germplasm 2,BAD,Wild,,,,,,, +,Germplasm 3,BAD1,Wild,,,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/circular_parent_dependencies.csv b/src/test/resources/files/germplasm_import/circular_parent_dependencies.csv index aca160536..eda6c2c49 100644 --- a/src/test/resources/files/germplasm_import/circular_parent_dependencies.csv +++ b/src/test/resources/files/germplasm_import/circular_parent_dependencies.csv @@ -1,4 +1,4 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germplasm 1,BCR,Wild,,,1,2,3,1234, -Germplasm 2,BCR,Wild,,,2,1,2,5678, -Germplasm 3,BCR,Wild,,,3,2,3,9123, \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,BCR,Wild,,,1,2,3,1234, +,Germplasm 2,BCR,Wild,,,2,1,2,5678, +,Germplasm 3,BCR,Wild,,,3,2,3,9123, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/duplicate_db_names.csv b/src/test/resources/files/germplasm_import/duplicate_db_names.csv index 29d16a241..0bdc5c5fe 100644 --- a/src/test/resources/files/germplasm_import/duplicate_db_names.csv +++ b/src/test/resources/files/germplasm_import/duplicate_db_names.csv @@ -1,7 +1,7 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Full Germplasm 1,ANE,Wild,,,,,,, -Full Germplasm 2,ANE,Wild,,,,,,, -Full Germplasm 3,ANE,Wild,,,,,,, -Unique Germplasm 1,ANE,Wild,,,,,,, -File Germplasm 1,ANE,Wild,,,,,,, -File Germplasm 1,ANE,Wild,,,,,,, \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Full Germplasm 1,ANE,Wild,,,,,,, +,Full Germplasm 2,ANE,Wild,,,,,,, +,Full Germplasm 3,ANE,Wild,,,,,,, +,Unique Germplasm 1,ANE,Wild,,,,,,, +,File Germplasm 1,ANE,Wild,,,,,,, +,File Germplasm 1,ANE,Wild,,,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/duplicate_entry_numbers.csv b/src/test/resources/files/germplasm_import/duplicate_entry_numbers.csv index 724731327..e20f5c845 100644 --- a/src/test/resources/files/germplasm_import/duplicate_entry_numbers.csv +++ b/src/test/resources/files/germplasm_import/duplicate_entry_numbers.csv @@ -1,5 +1,5 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germplasm 1,BCR,Wild,,,1,,,, -Germplasm 2,BCR,Wild,,,1,,,, -Germplasm 3,BCR,Wild,,,3,,,, -Germplasm 4,BCR,Wild,,,3,,,, \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,BCR,Wild,,,1,,,, +,Germplasm 2,BCR,Wild,,,1,,,, +,Germplasm 3,BCR,Wild,,,3,,,, +,Germplasm 4,BCR,Wild,,,3,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/empty_required_fields.csv b/src/test/resources/files/germplasm_import/empty_required_fields.csv index 0cb11e376..3072ce9b2 100644 --- a/src/test/resources/files/germplasm_import/empty_required_fields.csv +++ b/src/test/resources/files/germplasm_import/empty_required_fields.csv @@ -1,4 +1,4 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germplasm 1,ANE,Wild,,,,,,, -,ANE,Wild,,,,,,, -Germplasm 3,ANE,,,,,,,, \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,ANE,Wild,,,,,,, +,,ANE,Wild,,,,,,, +,Germplasm 3,ANE,,,,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/empty_values_required_fields.csv b/src/test/resources/files/germplasm_import/empty_values_required_fields.csv index 305016a6a..6af9e43a1 100644 --- a/src/test/resources/files/germplasm_import/empty_values_required_fields.csv +++ b/src/test/resources/files/germplasm_import/empty_values_required_fields.csv @@ -1,5 +1,5 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germplasm 1,,Wild,,,,,,, -,,Wild,,,,,,, -Germplasm 3,,,,,,,,, -,,Wild,,,,,,, \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,,Wild,,,,,,, +,,,Wild,,,,,,, +,Germplasm 3,,,,,,,,, +,,,Wild,,,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/female_dbid_not_exist.csv b/src/test/resources/files/germplasm_import/female_dbid_not_exist.csv index bdfaba7df..08a931520 100644 --- a/src/test/resources/files/germplasm_import/female_dbid_not_exist.csv +++ b/src/test/resources/files/germplasm_import/female_dbid_not_exist.csv @@ -1,4 +1,4 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germplasm 1,BCR,Wild,1000,,,,,, -Germplasm 2,BCR,Cultivated,1001,,,,,, -Germplasm 3,BCR,Kinda Wild,1002,,,,,, \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,BCR,Wild,1000,,,,,, +,Germplasm 2,BCR,Cultivated,1001,,,,,, +,Germplasm 3,BCR,Kinda Wild,1002,,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/female_entry_number_not_exist.csv b/src/test/resources/files/germplasm_import/female_entry_number_not_exist.csv index ad27c13f3..e8e3f4be4 100644 --- a/src/test/resources/files/germplasm_import/female_entry_number_not_exist.csv +++ b/src/test/resources/files/germplasm_import/female_entry_number_not_exist.csv @@ -1,4 +1,4 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germplasm 1,BCR,Wild,,,,1,,, -Germplasm 2,BCR,Cultivated,,,,2,,, -Germplasm 3,BCR,Kinda Wild,,,,3,,, \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,BCR,Wild,,,,1,,, +,Germplasm 2,BCR,Cultivated,,,,2,,, +,Germplasm 3,BCR,Kinda Wild,,,,3,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/full_import.csv b/src/test/resources/files/germplasm_import/full_import.csv index 88ce7645c..9073ac69f 100644 --- a/src/test/resources/files/germplasm_import/full_import.csv +++ b/src/test/resources/files/germplasm_import/full_import.csv @@ -1,4 +1,4 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Full Germplasm 1,ANE,Wild,1,2,2,,,1234,test1;test2 -Full Germplasm 2,ANE,Wild,2,,3,,,5678,test3;test2 -Full Germplasm 3,ANE,Wild,,,4,2,3,9123,test3;test4 \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Full Germplasm 1,ANE,Wild,1,2,2,,,1234,test1;test2 +,Full Germplasm 2,ANE,Wild,2,,3,,,5678,test3;test2 +,Full Germplasm 3,ANE,Wild,,,4,2,3,9123,test3;test4 \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/germplasm_column_casing.csv b/src/test/resources/files/germplasm_import/germplasm_column_casing.csv index 86c3b360f..f4c2e226a 100644 --- a/src/test/resources/files/germplasm_import/germplasm_column_casing.csv +++ b/src/test/resources/files/germplasm_import/germplasm_column_casing.csv @@ -1,2 +1,2 @@ -Name,BREEDING METHOD,SOURCE,fEmAlE pArEnT gId,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germ A,ANE,the shire,,,1,,,ABC-123, +GID,Name,BREEDING METHOD,SOURCE,fEmAlE pArEnT gId,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germ A,ANE,the shire,,,1,,,ABC-123, diff --git a/src/test/resources/files/germplasm_import/male_dbid_no_female_success.csv b/src/test/resources/files/germplasm_import/male_dbid_no_female_success.csv index 37625d690..23d9a47d8 100644 --- a/src/test/resources/files/germplasm_import/male_dbid_no_female_success.csv +++ b/src/test/resources/files/germplasm_import/male_dbid_no_female_success.csv @@ -1,4 +1,4 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germplasm 1,,Wild,,100,,,,, -Germplasm 2,,Cultivated,,101,,,,, -Germplasm 3,,Kinda Wild,,102,,,,, \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,,Wild,,100,,,,, +,Germplasm 2,,Cultivated,,101,,,,, +,Germplasm 3,,Kinda Wild,,102,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/male_dbid_not_exist.csv b/src/test/resources/files/germplasm_import/male_dbid_not_exist.csv index 65ee23994..f01b9a62d 100644 --- a/src/test/resources/files/germplasm_import/male_dbid_not_exist.csv +++ b/src/test/resources/files/germplasm_import/male_dbid_not_exist.csv @@ -1,4 +1,4 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germplasm 1,BCR,Wild,,100,,,,, -Germplasm 2,BCR,Cultivated,,101,,,,, -Germplasm 3,BCR,Kinda Wild,,102,,,,, \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,BCR,Wild,,100,,,,, +,Germplasm 2,BCR,Cultivated,,101,,,,, +,Germplasm 3,BCR,Kinda Wild,,102,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/male_entry_number_not_exist.csv b/src/test/resources/files/germplasm_import/male_entry_number_not_exist.csv index d0ca41fe7..bfa7f9c0d 100644 --- a/src/test/resources/files/germplasm_import/male_entry_number_not_exist.csv +++ b/src/test/resources/files/germplasm_import/male_entry_number_not_exist.csv @@ -1,4 +1,4 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germplasm 1,BCR,Wild,,,,,1,, -Germplasm 2,BCR,Cultivated,,,,,2,, -Germplasm 3,BCR,Kinda Wild,,,,,3,, \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,BCR,Wild,,,,,1,, +,Germplasm 2,BCR,Cultivated,,,,,2,, +,Germplasm 3,BCR,Kinda Wild,,,,,3,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/minimal_germplasm_import.csv b/src/test/resources/files/germplasm_import/minimal_germplasm_import.csv index 122209184..acb35b785 100644 --- a/src/test/resources/files/germplasm_import/minimal_germplasm_import.csv +++ b/src/test/resources/files/germplasm_import/minimal_germplasm_import.csv @@ -1,4 +1,4 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germplasm 1,BCR,Wild,,,,,,, -Germplasm 2,BCR,Cultivated,,,,,,, -Germplasm 3,BCR,Kinda Wild,,,,,,, \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,BCR,Wild,,,,,,, +,Germplasm 2,BCR,Cultivated,,,,,,, +,Germplasm 3,BCR,Kinda Wild,,,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/missing_optional_header.csv b/src/test/resources/files/germplasm_import/missing_optional_header.csv index 8ae96df57..0b1844cbb 100644 --- a/src/test/resources/files/germplasm_import/missing_optional_header.csv +++ b/src/test/resources/files/germplasm_import/missing_optional_header.csv @@ -1,4 +1,4 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germplasm 1,ANE,Wild,,,,,, -Germplasm 2,ANE,Wild,,,,,, -Germplasm 3,ANE,Wild,,,,,, \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,ANE,Wild,,,,,, +,Germplasm 2,ANE,Wild,,,,,, +,Germplasm 3,ANE,Wild,,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/missing_required_header.csv b/src/test/resources/files/germplasm_import/missing_required_header.csv index b1e28abda..584bfac1a 100644 --- a/src/test/resources/files/germplasm_import/missing_required_header.csv +++ b/src/test/resources/files/germplasm_import/missing_required_header.csv @@ -1,4 +1,4 @@ -Name,Breeding Method,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germplasm 1,,,,,,,, -Germplasm 2,,,,,,,, -Germplasm 3,,,,,,,, \ No newline at end of file +GID,Name,Breeding Method,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,,,,,,,, +,Germplasm 2,,,,,,,, +,Germplasm 3,,,,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/no_female_parent_blank_pedigree.csv b/src/test/resources/files/germplasm_import/no_female_parent_blank_pedigree.csv index 2e3ba2dbc..c3b06eee0 100644 --- a/src/test/resources/files/germplasm_import/no_female_parent_blank_pedigree.csv +++ b/src/test/resources/files/germplasm_import/no_female_parent_blank_pedigree.csv @@ -1,4 +1,4 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germplasm 1,BCR,Wild,,2,,,,1234, -Germplasm 2,BCR,Wild,,3,,,,5678, -Germplasm 3,BCR,Wild,,3,,,,9123, \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,BCR,Wild,,2,,,,1234, +,Germplasm 2,BCR,Wild,,3,,,,5678, +,Germplasm 3,BCR,Wild,,3,,,,9123, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/nonnumerical_entry_numbers.csv b/src/test/resources/files/germplasm_import/nonnumerical_entry_numbers.csv index 8bc876d69..fbca1ed3d 100644 --- a/src/test/resources/files/germplasm_import/nonnumerical_entry_numbers.csv +++ b/src/test/resources/files/germplasm_import/nonnumerical_entry_numbers.csv @@ -1,5 +1,5 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germplasm 1,,Wild,,,a,,,, -Germplasm 2,,Wild,,,b,,,, -Germplasm 3,,Wild,,,3,,,, -Germplasm 4,,Wild,,,4,,,, \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,,Wild,,,a,,,, +,Germplasm 2,,Wild,,,b,,,, +,Germplasm 3,,Wild,,,3,,,, +,Germplasm 4,,Wild,,,4,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/self_ref_parent_dependencies.csv b/src/test/resources/files/germplasm_import/self_ref_parent_dependencies.csv index 8f6e3de46..16144b020 100644 --- a/src/test/resources/files/germplasm_import/self_ref_parent_dependencies.csv +++ b/src/test/resources/files/germplasm_import/self_ref_parent_dependencies.csv @@ -1,4 +1,4 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germplasm 1,BCR,Wild,,,1,1,1,1234, -Germplasm 2,BCR,Wild,,,2,2,2,5678, -Germplasm 3,BCR,Wild,,,3,3,3,9123, \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,BCR,Wild,,,1,1,1,1234, +,Germplasm 2,BCR,Wild,,,2,2,2,5678, +,Germplasm 3,BCR,Wild,,,3,3,3,9123, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/some_entry_numbers.csv b/src/test/resources/files/germplasm_import/some_entry_numbers.csv index 77373d9cf..b86064c06 100644 --- a/src/test/resources/files/germplasm_import/some_entry_numbers.csv +++ b/src/test/resources/files/germplasm_import/some_entry_numbers.csv @@ -1,5 +1,5 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms -Germplasm 1,BCR,Wild,,,1,,,, -Germplasm 2,BCR,Wild,,,,,,, -Germplasm 3,BCR,Wild,,,3,,,, -Germplasm 4,BCR,Wild,,,,,,, \ No newline at end of file +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1,BCR,Wild,,,1,,,, +,Germplasm 2,BCR,Wild,,,,,,, +,Germplasm 3,BCR,Wild,,,3,,,, +,Germplasm 4,BCR,Wild,,,,,,, \ No newline at end of file From fcd2b9da0638e705dfc148ef53b645fe25d52d99 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Thu, 15 Sep 2022 15:20:50 -0400 Subject: [PATCH 03/23] Work in progress logic --- .../brapi/v2/dao/BrAPIGermplasmDAO.java | 1 + .../processors/ExperimentProcessor.java | 26 +++++---- .../processors/GermplasmProcessor.java | 58 ++++++++++++++++--- .../services/processors/Processor.java | 2 +- 4 files changed, 67 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index c8f42b303..0b8a36b97 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -212,6 +212,7 @@ private Map processGermplasmForDisplay(List importBrAPIGermplasm(List brAPIGermplasmList, UUID programId, ImportUpload upload) throws ApiException { GermplasmApi api = new GermplasmApi(programDAO.getCoreClient(programId)); + //todo try { Callable> postFunction = () -> brAPIDAOUtil.post(brAPIGermplasmList, upload, api::germplasmPost, importDAO::update); return programGermplasmCache.post(programId, postFunction); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 8f52a0a13..1e3220ae4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -65,18 +65,18 @@ public class ExperimentProcessor implements Processor { private String BRAPI_REFERENCE_SOURCE; - private DSLContext dsl; - private BrAPITrialDAO brapiTrialDAO; - private BrAPILocationDAO brAPILocationDAO; - private BrAPIStudyDAO brAPIStudyDAO; - private BrAPIObservationUnitDAO brAPIObservationUnitDAO; - private BrAPISeasonDAO brAPISeasonDAO; - private BrAPIGermplasmDAO brAPIGermplasmDAO; + private final DSLContext dsl; + private final BrAPITrialDAO brapiTrialDAO; + private final BrAPILocationDAO brAPILocationDAO; + private final BrAPIStudyDAO brAPIStudyDAO; + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private final BrAPISeasonDAO brAPISeasonDAO; + private final BrAPIGermplasmDAO brAPIGermplasmDAO; // used to make the yearsToSeasonDbId() function more efficient - private Map yearToSeasonDbIdCache = new HashMap<>(); + private final Map yearToSeasonDbIdCache = new HashMap<>(); // used to make the seasonDbIdtoYear() function more efficient - private Map seasonDbIdToYearCache = new HashMap<>(); + private final Map seasonDbIdToYearCache = new HashMap<>(); //These BrapiData-objects are initially populated by the getExistingBrapiData() method, // then updated by the getNewBrapiData() method. @@ -574,6 +574,8 @@ private Map> initialize_existingGerm .distinct() .collect(Collectors.toList()); + //todo check here + List existingGermplasms; try { existingGermplasms = this.getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId()); @@ -730,7 +732,7 @@ private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { } } catch (ApiException e) { - log.error(e.getResponseBody(), e);; + log.error(e.getResponseBody(), e); } String seasonDbId = (targetSeason==null) ? null : targetSeason.getSeasonDbId(); @@ -743,9 +745,9 @@ private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { try { season = this.brAPISeasonDAO.getSeasonById (seasonDbId, programId); } catch (ApiException e) { - log.error(e.getResponseBody(), e);; + log.error(e.getResponseBody(), e); } - Integer yearInt = (season == null) ? null : season.getYear();; + Integer yearInt = (season == null) ? null : season.getYear(); String yearStr = (yearInt==null) ? "" : yearInt.toString(); return yearStr; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index e6ace6a6e..728997578 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -27,8 +27,10 @@ import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.germ.BrAPIGermplasmSynonyms; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; import org.breedinginsight.brapps.importer.daos.BrAPIListDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; @@ -61,13 +63,15 @@ public class GermplasmProcessor implements Processor { private String BRAPI_REFERENCE_SOURCE; private BrAPIGermplasmService brAPIGermplasmService; - private BreedingMethodDAO breedingMethodDAO; - private BrAPIListDAO brAPIListDAO; - private DSLContext dsl; + private final BreedingMethodDAO breedingMethodDAO; + private final BrAPIListDAO brAPIListDAO; + private final DSLContext dsl; + private final BrAPIGermplasmDAO brAPIGermplasmDAO; Map> germplasmByAccessionNumber = new HashMap<>(); Map fileGermplasmByName = new HashMap<>(); Map dbGermplasmByName = new HashMap<>(); + Map dbGermplasmByAccessionNo = new HashMap<>(); Map germplasmIndexByEntryNo = new HashMap<>(); List newGermplasmList; List existingGermplasms; @@ -82,6 +86,8 @@ public class GermplasmProcessor implements Processor { public static String duplicateEntryNoMsg = "Entry numbers must be unique. Duplicated entry numbers found: %s"; public static String circularDependency = "Circular dependency in the pedigree tree"; public static String listNameAlreadyExists = "Import group name already exists"; + public static String missingGID = "No germplasm of GID %s was found in the database"; + public static String pedigreeAlreadyExists = "Pedigree information cannot be overwritten"; public static Function, String> arrayOfStringFormatter = (lst) -> { List lstCopy = new ArrayList<>(lst); Collections.sort(lstCopy); @@ -89,10 +95,11 @@ public class GermplasmProcessor implements Processor { }; @Inject - public GermplasmProcessor(BrAPIGermplasmService brAPIGermplasmService, DSLContext dsl, BreedingMethodDAO breedingMethodDAO, BrAPIListDAO brAPIListDAO) { + public GermplasmProcessor(BrAPIGermplasmService brAPIGermplasmService, DSLContext dsl, BreedingMethodDAO breedingMethodDAO, BrAPIListDAO brAPIListDAO, BrAPIGermplasmDAO brAPIGermplasmDAO) { this.brAPIGermplasmService = brAPIGermplasmService; this.dsl = dsl; this.breedingMethodDAO = breedingMethodDAO; + this.brAPIGermplasmDAO = brAPIGermplasmDAO; this.brAPIListDAO = brAPIListDAO; this.brAPIGermplasmService = brAPIGermplasmService; } @@ -148,13 +155,14 @@ public void getExistingBrapiData(List importRows, Program program) // Get existing germplasm names List dbGermplasm = brAPIGermplasmService.getGermplasmByDisplayName(new ArrayList<>(fileGermplasmByName.keySet()), program.getId()); dbGermplasm.stream().forEach(germplasm -> dbGermplasmByName.put(germplasm.getDefaultDisplayName(), germplasm)); + dbGermplasm.stream().forEach(germplasm -> dbGermplasmByAccessionNo.put(germplasm.getAccessionNumber(), germplasm)); // Check for existing germplasm lists Boolean listNameDup = false; if (importRows.size() > 0 && importRows.get(0).getGermplasm().getListName() != null) { try { Germplasm row = importRows.get(0).getGermplasm(); - String listName = row.constructGermplasmListName(row.getListName(), program); + String listName = Germplasm.constructGermplasmListName(row.getListName(), program); List existingLists = brAPIListDAO.getListByName(List.of(listName), program.getId()); for (BrAPIListSummary existingList: existingLists) { if (existingList.getListName().equals(listName)) { @@ -202,7 +210,7 @@ public void getExistingBrapiData(List importRows, Program program) @Override public Map process(List importRows, - Map mappedBrAPIImport, Program program, User user, boolean commit) throws ValidatorException { + Map mappedBrAPIImport, Program program, User user, boolean commit) throws ValidatorException, ApiException { // Method for generating accession number String germplasmSequenceName = program.getGermplasmSequence(); @@ -220,7 +228,7 @@ public Map process(List importRows importList = germplasm.constructBrAPIList(program, BRAPI_REFERENCE_SOURCE); } - // All rows are considered new germplasm, we don't check for duplicates + // All rows are considered new germplasm, we don't check for duplicates //todo except now we do oops newGermplasmList = new ArrayList<>(); Map breedingMethods = new HashMap<>(); Boolean nullEntryNotFound = false; @@ -235,7 +243,40 @@ public Map process(List importRows Germplasm germplasm = brapiImport.getGermplasm(); // Germplasm + + //todo double check what dbgermplasmbyaccessionNo actually getting + //TODO maybe make separate method for cleanliness if (germplasm != null && germplasm.getGermplasmName() != null) { + //Fetch and update existing germplasm + BrAPIGermplasm existingGermplasm; + if (germplasm.getAccessionNumber() != null) { + if (dbGermplasmByAccessionNo.containsKey(germplasm.getAccessionNumber()) ) { + existingGermplasm = dbGermplasmByAccessionNo.get(germplasm.getAccessionNumber()); + } else { + ValidationError ve = new ValidationError("GID", missingGID, HttpStatus.NOT_FOUND); + validationErrors.addError(i+2, ve ); // +2 instead of +1 to account for the column header row. + continue; + } + if (germplasm.getFemaleParentDBID() != null || germplasm.getFemaleParentEntryNo() != null) { + if (germplasm.getFemaleParentDBID() == null && germplasm.getFemaleParentEntryNo() == null) { + //todo handle later cause gonna get weird in mixed file with posting order + } else { + ValidationError ve = new ValidationError("Pedigree", pedigreeAlreadyExists, HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(i+2, ve ); // +2 instead of +1 to account for the column header row. + continue; + } + } + if (germplasm.getSynonyms() != null) { + for (String synonym: germplasm.getSynonyms().split(";")){ + BrAPIGermplasmSynonyms newSynonym = new BrAPIGermplasmSynonyms(); + newSynonym.setSynonym(synonym); + existingGermplasm.addSynonymsItem(newSynonym); + } + } + + //Consider case with only edits, only new, and mixed + //check to see how this should be handled relative to newGermplasmList + } // Get the breeding method database object BreedingMethodEntity breedingMethod = null; @@ -308,6 +349,7 @@ public Map process(List importRows } // Construct our response object + //todo handle stats ImportPreviewStatistics germplasmStats = ImportPreviewStatistics.builder() .newObjectCount(newGermplasmList.size()) .build(); @@ -331,6 +373,8 @@ private void createPostOrder() { // Construct a dependency tree for POSTing order Set created = existingGermplasms.stream().map(BrAPIGermplasm::getGermplasmName).collect(Collectors.toSet()); + //todo this gets messy + int totalRecorded = 0; while (totalRecorded < newGermplasmList.size()) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java index 16cff931c..ee643596a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java @@ -49,7 +49,7 @@ public interface Processor { */ Map process(List importRows, Map mappedBrAPIImport, - Program program, User user, boolean commit) throws ValidatorException; + Program program, User user, boolean commit) throws ValidatorException, ApiException; /** * Given mapped brapi import with updates from prior dependencies, check if have everything needed From c580f0bd6909f6c47c69526f665d36d3514049de Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Thu, 2 Feb 2023 16:36:56 -0500 Subject: [PATCH 04/23] Updated migration version number --- ...m_import_add_GID.sql => V1.0.12__germplasm_import_add_GID.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V0.7.2__germplasm_import_add_GID.sql => V1.0.12__germplasm_import_add_GID.sql} (100%) diff --git a/src/main/resources/db/migration/V0.7.2__germplasm_import_add_GID.sql b/src/main/resources/db/migration/V1.0.12__germplasm_import_add_GID.sql similarity index 100% rename from src/main/resources/db/migration/V0.7.2__germplasm_import_add_GID.sql rename to src/main/resources/db/migration/V1.0.12__germplasm_import_add_GID.sql From 1928ef337130c8c069cd6539635f385b90d9a207 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Tue, 28 Feb 2023 16:59:58 -0500 Subject: [PATCH 05/23] Updates to processing logic --- .../processors/GermplasmProcessor.java | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index e28377df5..0908bec72 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -253,9 +253,11 @@ public Map process(List importRows //todo double check what dbgermplasmbyaccessionNo actually getting //TODO maybe make separate method for cleanliness - if (germplasm != null && germplasm.getGermplasmName() != null) { + if (germplasm != null) { //Fetch and update existing germplasm BrAPIGermplasm existingGermplasm; + + // Have GID so updating an existing germplasm record if (germplasm.getAccessionNumber() != null) { if (dbGermplasmByAccessionNo.containsKey(germplasm.getAccessionNumber()) ) { existingGermplasm = dbGermplasmByAccessionNo.get(germplasm.getAccessionNumber()); @@ -264,6 +266,17 @@ public Map process(List importRows validationErrors.addError(i+2, ve ); // +2 instead of +1 to account for the column header row. continue; } + + // Has an existing pedigree so add error and skip + if (!StringUtils.isBlank(existingGermplasm.getPedigree())) { + ValidationError ve = new ValidationError("Pedigree", pedigreeAlreadyExists, HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(i+2, ve ); // +2 instead of +1 to account for the column header row. + continue; + } + + + + /* if (germplasm.getFemaleParentDBID() != null || germplasm.getFemaleParentEntryNo() != null) { if (germplasm.getFemaleParentDBID() == null && germplasm.getFemaleParentEntryNo() == null) { //todo handle later cause gonna get weird in mixed file with posting order @@ -273,11 +286,18 @@ public Map process(List importRows continue; } } + */ + + // Append synonyms to germplasm that don't already exist + // Synonym comparison is based on name and type if (germplasm.getSynonyms() != null) { + Set existingSynonyms = new HashSet<>(existingGermplasm.getSynonyms()); for (String synonym: germplasm.getSynonyms().split(";")){ - BrAPIGermplasmSynonyms newSynonym = new BrAPIGermplasmSynonyms(); - newSynonym.setSynonym(synonym); - existingGermplasm.addSynonymsItem(newSynonym); + BrAPIGermplasmSynonyms brapiSynonym = new BrAPIGermplasmSynonyms(); + brapiSynonym.setSynonym(synonym); + if (!existingSynonyms.contains(brapiSynonym)) { + existingGermplasm.addSynonymsItem(brapiSynonym); + } } } From 1c03d641d64839c8c29ddff5e7f0be35edaa3e38 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Thu, 9 Mar 2023 10:59:07 -0500 Subject: [PATCH 06/23] Brapi data updates work --- .../api/v1/controller/TokenController.java | 27 ++- .../brapi/v2/dao/BrAPIGermplasmDAO.java | 19 ++- .../v2/services/BrAPIGermplasmService.java | 5 +- .../processors/GermplasmProcessor.java | 161 +++++++++++------- .../utilities/BrAPIDAOUtil.java | 25 +++ 5 files changed, 168 insertions(+), 69 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java b/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java index e1356dbfd..5f21a29f4 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java @@ -17,8 +17,10 @@ package org.breedinginsight.api.v1.controller; import io.micronaut.http.HttpResponse; +import io.micronaut.http.MediaType; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Get; +import io.micronaut.http.annotation.Produces; import io.micronaut.http.annotation.QueryValue; import io.micronaut.http.uri.UriBuilder; import io.micronaut.security.annotation.Secured; @@ -26,13 +28,17 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.auth.SecurityService; +import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.ApiToken; +import org.breedinginsight.model.User; import org.breedinginsight.services.TokenService; import javax.inject.Inject; import javax.validation.constraints.NotBlank; import java.net.URI; +import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; @Slf4j @Controller("/${micronaut.bi.api.version}") @@ -41,10 +47,13 @@ public class TokenController { private SecurityService securityService; private TokenService tokenService; + private UserDAO userDAO; + @Inject - public TokenController(SecurityService securityService, TokenService tokenService) { + public TokenController(SecurityService securityService, TokenService tokenService, UserDAO userDAO) { this.securityService = securityService; this.tokenService = tokenService; + this.userDAO = userDAO; } @Get("/api-token") @@ -71,4 +80,18 @@ public HttpResponse apiToken(@QueryValue @NotBlank String returnUrl) { } -} + @Get("/dev-token") + @Produces(MediaType.APPLICATION_JSON) + @Secured(SecurityRule.IS_ANONYMOUS) + public HttpResponse devToken() { + + User user = userDAO.getUserByOrcId("0000-0003-0437-8310").get(); + List systemRoles = user.getSystemRoles().stream() + .map(systemRole -> systemRole.getDomain().toUpperCase()).collect(Collectors.toList()); + AuthenticatedUser actingUser = new AuthenticatedUser(user.getName(), + systemRoles, user.getId(), user.getProgramRoles()); + + return HttpResponse.ok(tokenService.generateApiToken(actingUser)); + } + +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index 231c04444..6056955a1 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -234,14 +234,23 @@ private Map processGermplasmForDisplay(List importBrAPIGermplasm(List brAPIGermplasmList, UUID programId, ImportUpload upload) throws ApiException { + public List importBrAPIGermplasm(List postBrAPIGermplasmList, List putBrAPIGermplasmList, + UUID programId, ImportUpload upload) throws ApiException { GermplasmApi api = new GermplasmApi(programDAO.getCoreClient(programId)); var program = programDAO.fetchOneById(programId); + Callable> postFunction; try { - Callable> postFunction = () -> { - List postResponse = brAPIDAOUtil.post(brAPIGermplasmList, upload, api::germplasmPost, importDAO::update); - return processGermplasmForDisplay(postResponse, program.getKey()); - }; + if (!postBrAPIGermplasmList.isEmpty()) { + postFunction = () -> { + List postResponse = brAPIDAOUtil.post(postBrAPIGermplasmList, upload, api::germplasmPost, importDAO::update); + return processGermplasmForDisplay(postResponse, program.getKey()); + }; + } else { + postFunction = () -> { + List postResponse = brAPIDAOUtil.putGermplasm(putBrAPIGermplasmList, api); + return processGermplasmForDisplay(postResponse, program.getKey()); + }; + } return programGermplasmCache.post(programId, postFunction); } catch (Exception e) { throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java index b7c4fa323..c953fd3b4 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java @@ -287,8 +287,9 @@ private String removeAppendedKey(String listName, String programKey){ return listName.replace(appendedKey, ""); } - public List importBrAPIGermplasm(List brAPIGermplasmList, UUID programId, ImportUpload upload) throws ApiException { - return germplasmDAO.importBrAPIGermplasm(brAPIGermplasmList, programId, upload); + public List importBrAPIGermplasm(List postBrAPIGermplasmList, List putBrAPIGermplasmList, + UUID programId, ImportUpload upload) throws ApiException { + return germplasmDAO.importBrAPIGermplasm(postBrAPIGermplasmList, putBrAPIGermplasmList, programId, upload); } public List getRawGermplasmByAccessionNumber(ArrayList germplasmAccessionNumbers, UUID programId) throws ApiException { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index 836430a1e..aaed50256 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -16,6 +16,9 @@ */ package org.breedinginsight.brapps.importer.services.processors; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.HttpStatus; @@ -30,6 +33,7 @@ import org.brapi.v2.model.germ.BrAPIGermplasmSynonyms; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; import org.breedinginsight.brapps.importer.daos.BrAPIListDAO; @@ -75,6 +79,8 @@ public class GermplasmProcessor implements Processor { Map dbGermplasmByAccessionNo = new HashMap<>(); Map germplasmIndexByEntryNo = new HashMap<>(); List newGermplasmList; + + List updatedGermplasmList; List existingGermplasms; List existingParentGermplasms; List> postOrder = new ArrayList<>(); @@ -237,6 +243,7 @@ public Map process(List importRows // All rows are considered new germplasm, we don't check for duplicates //todo except now we do oops newGermplasmList = new ArrayList<>(); + updatedGermplasmList = new ArrayList<>(); Map breedingMethods = new HashMap<>(); Boolean nullEntryNotFound = false; List badBreedingMethods = new ArrayList<>(); @@ -249,6 +256,17 @@ public Map process(List importRows Germplasm germplasm = brapiImport.getGermplasm(); + // Assign the entry number + if (germplasm.getEntryNo() == null) { + germplasm.setEntryNo(Integer.toString(i + 1)); + } else { + userProvidedEntryNumbers.add(germplasm.getEntryNo()); + } + entryNumberCounts.put(germplasm.getEntryNo(), + entryNumberCounts.containsKey(germplasm.getEntryNo()) ? entryNumberCounts.get(germplasm.getEntryNo()) + 1 : 1); + + UUID importListId = brAPIGermplasmService.getGermplasmListId(importList); + // Germplasm //todo double check what dbgermplasmbyaccessionNo actually getting @@ -268,26 +286,15 @@ public Map process(List importRows } // Has an existing pedigree so add error and skip + // no pedigree on second import, don't change existing + // if there is pedigree & previously not, upgrade pedigree + // if previous pedigree & different pedigree, validation error if (!StringUtils.isBlank(existingGermplasm.getPedigree())) { ValidationError ve = new ValidationError("Pedigree", pedigreeAlreadyExists, HttpStatus.UNPROCESSABLE_ENTITY); validationErrors.addError(i+2, ve ); // +2 instead of +1 to account for the column header row. continue; } - - - /* - if (germplasm.getFemaleParentDBID() != null || germplasm.getFemaleParentEntryNo() != null) { - if (germplasm.getFemaleParentDBID() == null && germplasm.getFemaleParentEntryNo() == null) { - //todo handle later cause gonna get weird in mixed file with posting order - } else { - ValidationError ve = new ValidationError("Pedigree", pedigreeAlreadyExists, HttpStatus.UNPROCESSABLE_ENTITY); - validationErrors.addError(i+2, ve ); // +2 instead of +1 to account for the column header row. - continue; - } - } - */ - // Append synonyms to germplasm that don't already exist // Synonym comparison is based on name and type if (germplasm.getSynonyms() != null) { @@ -301,53 +308,54 @@ public Map process(List importRows } } - //Consider case with only edits, only new, and mixed - //check to see how this should be handled relative to newGermplasmList - } + validatePedigree(germplasm, i+2, validationErrors); - // Get the breeding method database object - ProgramBreedingMethodEntity breedingMethod = null; - if (germplasm.getBreedingMethod() != null) { - if (breedingMethods.containsKey(germplasm.getBreedingMethod())) { - breedingMethod = breedingMethods.get(germplasm.getBreedingMethod()); - } else { - List breedingMethodResults = breedingMethodDAO.findByNameOrAbbreviation(germplasm.getBreedingMethod(), program.getId()); - if (breedingMethodResults.size() > 0) { - breedingMethods.put(germplasm.getBreedingMethod(), breedingMethodResults.get(0)); + // Add germplasm to the new list + JsonObject listEntryNumbers = existingGermplasm.getAdditionalInfo().getAsJsonObject(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ENTRY_NUMBERS); + listEntryNumbers.addProperty(importListId.toString(), germplasm.getEntryNo()); + + // TODO: figure out why clear this out + existingGermplasm.setBreedingMethodDbId(null); + + updatedGermplasmList.add(existingGermplasm); + mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm)); + + importList.addDataItem(existingGermplasm.getGermplasmName()); + + } else { + // Get the breeding method database object + ProgramBreedingMethodEntity breedingMethod = null; + if (germplasm.getBreedingMethod() != null) { + if (breedingMethods.containsKey(germplasm.getBreedingMethod())) { breedingMethod = breedingMethods.get(germplasm.getBreedingMethod()); } else { - ValidationError ve = new ValidationError("Breeding Method", badBreedMethodsMsg, HttpStatus.UNPROCESSABLE_ENTITY); - validationErrors.addError(i+2, ve ); // +2 instead of +1 to account for the column header row. - badBreedingMethods.add(germplasm.getBreedingMethod()); - breedingMethod = null; + List breedingMethodResults = breedingMethodDAO.findByNameOrAbbreviation(germplasm.getBreedingMethod(), program.getId()); + if (breedingMethodResults.size() > 0) { + breedingMethods.put(germplasm.getBreedingMethod(), breedingMethodResults.get(0)); + breedingMethod = breedingMethods.get(germplasm.getBreedingMethod()); + } else { + ValidationError ve = new ValidationError("Breeding Method", badBreedMethodsMsg, HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(i + 2, ve); // +2 instead of +1 to account for the column header row. + badBreedingMethods.add(germplasm.getBreedingMethod()); + breedingMethod = null; + } } } - } - - validatePedigree(germplasm, i+2, validationErrors); - // Assign the entry number - if (germplasm.getEntryNo() == null) { - germplasm.setEntryNo(Integer.toString(i + 1)); - } else { - userProvidedEntryNumbers.add(germplasm.getEntryNo()); - } - entryNumberCounts.put(germplasm.getEntryNo(), - entryNumberCounts.containsKey(germplasm.getEntryNo()) ? entryNumberCounts.get(germplasm.getEntryNo()) + 1 : 1); + validatePedigree(germplasm, i + 2, validationErrors); - UUID importListId = brAPIGermplasmService.getGermplasmListId(importList); + BrAPIGermplasm newGermplasm = germplasm.constructBrAPIGermplasm(program, breedingMethod, user, commit, BRAPI_REFERENCE_SOURCE, nextVal, importListId); - BrAPIGermplasm newGermplasm = germplasm.constructBrAPIGermplasm(program, breedingMethod, user, commit, BRAPI_REFERENCE_SOURCE, nextVal, importListId); + newGermplasmList.add(newGermplasm); + // Assign status of the germplasm + if (fileGermplasmByName.get(newGermplasm.getDefaultDisplayName()) > 1 || dbGermplasmByName.containsKey(newGermplasm.getDefaultDisplayName())) { + mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.EXISTING, newGermplasm)); + } else { + mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.NEW, newGermplasm)); + } - newGermplasmList.add(newGermplasm); - // Assign status of the germplasm - if (fileGermplasmByName.get(newGermplasm.getDefaultDisplayName()) > 1 || dbGermplasmByName.containsKey(newGermplasm.getDefaultDisplayName())) { - mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.EXISTING, newGermplasm)); - } else { - mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.NEW, newGermplasm)); + importList.addDataItem(newGermplasm.getGermplasmName()); } - - importList.addDataItem(newGermplasm.getGermplasmName()); } else { mappedImportRow.setGermplasm(null); } @@ -380,18 +388,33 @@ public Map process(List importRows } // Construct our response object - //todo handle stats + return getStatisticsMap(importRows); + } + + private void processUpdatedGermplasm() { + + } + + private Map getStatisticsMap(List importRows) { + ImportPreviewStatistics germplasmStats = ImportPreviewStatistics.builder() - .newObjectCount(newGermplasmList.size()) - .build(); + .newObjectCount(newGermplasmList.size()) + .ignoredObjectCount(germplasmByAccessionNumber.size()) + .build(); + + /* + .newObjectCount(ProcessorData.getNumNewObjects(observationByHash)) + .ignoredObjectCount(ProcessorData.getNumExistingObjects(observationByHash)) + */ + //Modified logic here to check for female parent dbid or entry no, removed check for male due to assumption that shouldn't have only male parent int newObjectCount = newGermplasmList.stream().filter(newGermplasm -> newGermplasm != null).collect(Collectors.toList()).size(); ImportPreviewStatistics pedigreeConnectStats = ImportPreviewStatistics.builder() - .newObjectCount(importRows.stream().filter(germplasmImport -> - germplasmImport.getGermplasm() != null && - (germplasmImport.getGermplasm().getFemaleParentDBID() != null || germplasmImport.getGermplasm().getFemaleParentEntryNo() != null) - ).collect(Collectors.toList()).size()).build(); + .newObjectCount(importRows.stream().filter(germplasmImport -> + germplasmImport.getGermplasm() != null && + (germplasmImport.getGermplasm().getFemaleParentDBID() != null || germplasmImport.getGermplasm().getFemaleParentEntryNo() != null) + ).collect(Collectors.toList()).size()).build(); return Map.of( "Germplasm", germplasmStats, @@ -470,12 +493,29 @@ public void postBrapiData(Map mappedBrAPIImport, Program // POST Germplasm List createdGermplasm = new ArrayList<>(); - if (newGermplasmList.size() > 0) { + + if (!newGermplasmList.isEmpty()) { try { for (List postGroup: postOrder){ - createdGermplasm.addAll(brAPIGermplasmService.importBrAPIGermplasm(postGroup, program.getId(), upload)); + createdGermplasm.addAll(brAPIGermplasmService.importBrAPIGermplasm(postGroup, new ArrayList<>(), program.getId(), upload)); } + } catch (ApiException e) { + throw new InternalServerException(e.toString(), e); + } + } + + // PUT germplasm + if (!updatedGermplasmList.isEmpty()) { + try { + brAPIGermplasmService.importBrAPIGermplasm(new ArrayList<>(), updatedGermplasmList, program.getId(), upload); + } catch (ApiException e) { + throw new InternalServerException(e.toString(), e); + } + } + // Create list + if (!newGermplasmList.isEmpty() || !updatedGermplasmList.isEmpty()) { + try { // Create germplasm list brAPIListDAO.createBrAPILists(List.of(importList), program.getId(), upload); } catch (ApiException e) { @@ -483,6 +523,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program } } + // Update our records with what is returned Map createdGermplasmMap = new HashMap<>(); createdGermplasm.forEach(germplasm -> { diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index 60a6789de..a1f8335b7 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -27,7 +27,10 @@ import org.apache.commons.lang3.tuple.Pair; import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.modules.germplasm.GermplasmApi; import org.brapi.v2.model.*; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.germ.response.BrAPIGermplasmSingleResponse; import org.breedinginsight.brapps.importer.model.ImportUpload; import javax.inject.Singleton; @@ -168,6 +171,28 @@ private List getListResult(ApiResponse, Optional(); } + // TODO: write generic put code + public List put(List brapiObjects, + ImportUpload upload, + Function, ApiResponse> putMethod, + Consumer progressUpdateMethod) throws ApiException { + + return new ArrayList<>(); + } + + // TODO: temporary until generic code is written + public List putGermplasm(List germplasmList, GermplasmApi api) throws ApiException { + List listResult = new ArrayList<>(); + + for (BrAPIGermplasm germplasm : germplasmList) { + ApiResponse response = api.germplasmGermplasmDbIdPut(germplasm.getGermplasmDbId(), germplasm); + listResult.add(response.getBody().getResult()); + } + + return listResult; + } + + public List post(List brapiObjects, ImportUpload upload, Function, ApiResponse> postMethod, From 3282d52d57e09b462aa4f9be849d01863c4c9ba6 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Thu, 16 Mar 2023 10:51:40 -0400 Subject: [PATCH 07/23] Some cleanup and bug fixes --- .../brapi/v2/dao/BrAPIGermplasmDAO.java | 16 ++++++- .../brapps/importer/model/base/Germplasm.java | 44 +++++++++++++++++++ .../processors/GermplasmProcessor.java | 38 +++++++--------- .../utilities/BrAPIDAOUtil.java | 13 ------ 4 files changed, 74 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index 6056955a1..81eae6440 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -32,6 +32,7 @@ import org.brapi.v2.model.germ.BrAPIGermplasmSynonyms; import org.brapi.v2.model.germ.request.BrAPIGermplasmSearchRequest; import org.brapi.v2.model.germ.response.BrAPIGermplasmListResponse; +import org.brapi.v2.model.germ.response.BrAPIGermplasmSingleResponse; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; @@ -247,7 +248,7 @@ public List importBrAPIGermplasm(List postBrAPIG }; } else { postFunction = () -> { - List postResponse = brAPIDAOUtil.putGermplasm(putBrAPIGermplasmList, api); + List postResponse = putGermplasm(putBrAPIGermplasmList, api); return processGermplasmForDisplay(postResponse, program.getKey()); }; } @@ -289,4 +290,17 @@ public BrAPIGermplasm getGermplasmByDBID(String germplasmDbId, UUID programId) t } return germplasm; } + + public List putGermplasm(List germplasmList, GermplasmApi api) throws ApiException { + List listResult = new ArrayList<>(); + + // TODO: temporary until generic BrAPIDAOUtil code is written + // generic code should handle importer progress updates + for (BrAPIGermplasm germplasm : germplasmList) { + ApiResponse response = api.germplasmGermplasmDbIdPut(germplasm.getGermplasmDbId(), germplasm); + listResult.add(response.getBody().getResult()); + } + + return listResult; + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java index b9f37d68b..f6bd51fe3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java @@ -17,6 +17,7 @@ package org.breedinginsight.brapps.importer.model.base; +import com.google.gson.JsonObject; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -153,6 +154,49 @@ public static String constructGermplasmListName(String listName, Program program return String.format("%s [%s-germplasm]", listName, program.getKey()); } + public void updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID listId, boolean commit) { + + germplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_GID, getFemaleParentDBID()); + germplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID, getMaleParentDBID()); + germplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_ENTRY_NO, getFemaleParentEntryNo()); + germplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_ENTRY_NO, getMaleParentEntryNo()); + + // Append synonyms to germplasm that don't already exist + // Synonym comparison is based on name and type + if (synonyms != null) { + Set existingSynonyms = new HashSet<>(germplasm.getSynonyms()); + for (String synonym: synonyms.split(";")){ + BrAPIGermplasmSynonyms brapiSynonym = new BrAPIGermplasmSynonyms(); + brapiSynonym.setSynonym(synonym); + if (!existingSynonyms.contains(brapiSynonym)) { + germplasm.addSynonymsItem(brapiSynonym); + } + } + } + + // Add germplasm to the new list + JsonObject listEntryNumbers = germplasm.getAdditionalInfo().getAsJsonObject(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ENTRY_NUMBERS); + listEntryNumbers.addProperty(listId.toString(), entryNo); + + // TODO: figure out why clear this out: brapi-server + germplasm.setBreedingMethodDbId(null); + + if (commit) { + setUpdateCommitFields(germplasm, program.getKey()); + } + } + + + public void setUpdateCommitFields(BrAPIGermplasm germplasm, String programKey) { + // Update our synonyms to [-] + if (germplasm.getSynonyms() != null && !germplasm.getSynonyms().isEmpty()) { + for (BrAPIGermplasmSynonyms synonym: germplasm.getSynonyms()) { + synonym.setSynonym(Utilities.appendProgramKey(synonym.getSynonym(), programKey, germplasm.getAccessionNumber())); + } + } + } + + public BrAPIGermplasm constructBrAPIGermplasm(ProgramBreedingMethodEntity breedingMethod, User user, UUID listId) { BrAPIGermplasm germplasm = new BrAPIGermplasm(); germplasm.setGermplasmName(getGermplasmName()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index aaed50256..651851a46 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -295,27 +295,9 @@ public Map process(List importRows continue; } - // Append synonyms to germplasm that don't already exist - // Synonym comparison is based on name and type - if (germplasm.getSynonyms() != null) { - Set existingSynonyms = new HashSet<>(existingGermplasm.getSynonyms()); - for (String synonym: germplasm.getSynonyms().split(";")){ - BrAPIGermplasmSynonyms brapiSynonym = new BrAPIGermplasmSynonyms(); - brapiSynonym.setSynonym(synonym); - if (!existingSynonyms.contains(brapiSynonym)) { - existingGermplasm.addSynonymsItem(brapiSynonym); - } - } - } - validatePedigree(germplasm, i+2, validationErrors); - // Add germplasm to the new list - JsonObject listEntryNumbers = existingGermplasm.getAdditionalInfo().getAsJsonObject(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ENTRY_NUMBERS); - listEntryNumbers.addProperty(importListId.toString(), germplasm.getEntryNo()); - - // TODO: figure out why clear this out - existingGermplasm.setBreedingMethodDbId(null); + germplasm.updateBrAPIGermplasm(existingGermplasm, program, importListId, commit); updatedGermplasmList.add(existingGermplasm); mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm)); @@ -608,10 +590,20 @@ else if (germplasmIndexByEntryNo.containsKey(germplasm.getFemaleParentEntryNo()) } } } - mappedBrAPIImport.get(i).getGermplasm().getBrAPIObject().setPedigree(pedigreeString.length() > 0 ? pedigreeString.toString() : null); - //Simpler to just always add boolean, but consider for logic that previous imported values won't have that additional info value - mappedBrAPIImport.get(i).getGermplasm().getBrAPIObject().putAdditionalInfoItem("femaleParentUnknown", femaleParentUnknown); - mappedBrAPIImport.get(i).getGermplasm().getBrAPIObject().putAdditionalInfoItem("maleParentUnknown", maleParentUnknown); + + // only update brapi object for new germplasm or update with no previous pedigree + BrAPIGermplasm brapiGermplasm = mappedBrAPIImport.get(i).getGermplasm().getBrAPIObject(); + + Optional existingPedigree = existingGermplasms.stream() + .filter(g -> g.equals(brapiGermplasm) && StringUtils.isBlank(g.getPedigree())) + .findFirst(); + + if (existingPedigree.isEmpty()) { + mappedBrAPIImport.get(i).getGermplasm().getBrAPIObject().setPedigree(pedigreeString.length() > 0 ? pedigreeString.toString() : null); + //Simpler to just always add boolean, but consider for logic that previous imported values won't have that additional info value + mappedBrAPIImport.get(i).getGermplasm().getBrAPIObject().putAdditionalInfoItem("femaleParentUnknown", femaleParentUnknown); + mappedBrAPIImport.get(i).getGermplasm().getBrAPIObject().putAdditionalInfoItem("maleParentUnknown", maleParentUnknown); + } } } } diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index a1f8335b7..3fa26b963 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -180,19 +180,6 @@ public List put(List brapiObjects, return new ArrayList<>(); } - // TODO: temporary until generic code is written - public List putGermplasm(List germplasmList, GermplasmApi api) throws ApiException { - List listResult = new ArrayList<>(); - - for (BrAPIGermplasm germplasm : germplasmList) { - ApiResponse response = api.germplasmGermplasmDbIdPut(germplasm.getGermplasmDbId(), germplasm); - listResult.add(response.getBody().getResult()); - } - - return listResult; - } - - public List post(List brapiObjects, ImportUpload upload, Function, ApiResponse> postMethod, From 8f802608ea594db939acf9d633bda447a996ac34 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Thu, 16 Mar 2023 11:18:25 -0400 Subject: [PATCH 08/23] Remove dev token --- .../api/v1/controller/TokenController.java | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java b/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java index 5f21a29f4..d3d177fc1 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java @@ -17,10 +17,8 @@ package org.breedinginsight.api.v1.controller; import io.micronaut.http.HttpResponse; -import io.micronaut.http.MediaType; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Get; -import io.micronaut.http.annotation.Produces; import io.micronaut.http.annotation.QueryValue; import io.micronaut.http.uri.UriBuilder; import io.micronaut.security.annotation.Secured; @@ -30,15 +28,12 @@ import org.breedinginsight.api.auth.SecurityService; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.ApiToken; -import org.breedinginsight.model.User; import org.breedinginsight.services.TokenService; import javax.inject.Inject; import javax.validation.constraints.NotBlank; import java.net.URI; -import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; @Slf4j @Controller("/${micronaut.bi.api.version}") @@ -80,18 +75,4 @@ public HttpResponse apiToken(@QueryValue @NotBlank String returnUrl) { } - @Get("/dev-token") - @Produces(MediaType.APPLICATION_JSON) - @Secured(SecurityRule.IS_ANONYMOUS) - public HttpResponse devToken() { - - User user = userDAO.getUserByOrcId("0000-0003-0437-8310").get(); - List systemRoles = user.getSystemRoles().stream() - .map(systemRole -> systemRole.getDomain().toUpperCase()).collect(Collectors.toList()); - AuthenticatedUser actingUser = new AuthenticatedUser(user.getName(), - systemRoles, user.getId(), user.getProgramRoles()); - - return HttpResponse.ok(tokenService.generateApiToken(actingUser)); - } - } \ No newline at end of file From b2bc961b4a404265924122573dc0164ca1e047d9 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Thu, 16 Mar 2023 11:40:18 -0400 Subject: [PATCH 09/23] Cleanup left over token stuff --- .../org/breedinginsight/api/v1/controller/TokenController.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java b/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java index d3d177fc1..4823e65d0 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java @@ -45,10 +45,9 @@ public class TokenController { private UserDAO userDAO; @Inject - public TokenController(SecurityService securityService, TokenService tokenService, UserDAO userDAO) { + public TokenController(SecurityService securityService, TokenService tokenService) { this.securityService = securityService; this.tokenService = tokenService; - this.userDAO = userDAO; } @Get("/api-token") From 67c9fe8e3f5899895db2569bf2e87d9578854619 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Thu, 16 Mar 2023 17:00:02 -0400 Subject: [PATCH 10/23] Renamed migration after develop merge --- ...m_import_add_GID.sql => V1.0.13__germplasm_import_add_GID.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V1.0.12__germplasm_import_add_GID.sql => V1.0.13__germplasm_import_add_GID.sql} (100%) diff --git a/src/main/resources/db/migration/V1.0.12__germplasm_import_add_GID.sql b/src/main/resources/db/migration/V1.0.13__germplasm_import_add_GID.sql similarity index 100% rename from src/main/resources/db/migration/V1.0.12__germplasm_import_add_GID.sql rename to src/main/resources/db/migration/V1.0.13__germplasm_import_add_GID.sql From 1bd12bf34356d3b1e00aceaa1db6d785913444d5 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Fri, 17 Mar 2023 14:25:45 -0400 Subject: [PATCH 11/23] Fixed compilation issues --- .../controller/BreedingMethodControllerIntegrationTest.java | 6 +++--- .../brapps/importer/ExperimentFileImportTest.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java index cf793f5bf..8336a8d16 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java @@ -400,7 +400,7 @@ public void createGermplasmWithProgramMethod() throws ApiException { .externalReferences(List.of(programRef, germIdRef)) .accessionNumber(accessionNum); - assertDoesNotThrow(() -> germplasmService.importBrAPIGermplasm(List.of(germplasm), program.getId(), null)); + assertDoesNotThrow(() -> germplasmService.importBrAPIGermplasm(List.of(germplasm), new ArrayList<>(), program.getId(), null)); String germplasmUrl = String.format("/programs/%s/brapi/v2/germplasm", program.getId()); Flowable> germplasmCall = client.exchange( @@ -472,7 +472,7 @@ public void tryDeleteProgramMethodInUse() throws ApiException { .externalReferences(List.of(programRef, germIdRef)) .accessionNumber(accessionNum); - assertDoesNotThrow(() -> germplasmService.importBrAPIGermplasm(List.of(germplasm), program.getId(), null)); + assertDoesNotThrow(() -> germplasmService.importBrAPIGermplasm(List.of(germplasm), new ArrayList<>(), program.getId(), null)); String deleteUrl = String.format("/programs/%s/breeding-methods/%s", program.getId(), createdMethod.getId()); Flowable> deleteCall = client.exchange( @@ -548,7 +548,7 @@ public void tryDisableSystemMethodInUse() { .externalReferences(List.of(programRef, germIdRef)) .accessionNumber(accessionNum); - assertDoesNotThrow(() -> germplasmService.importBrAPIGermplasm(List.of(germplasm), program.getId(), null)); + assertDoesNotThrow(() -> germplasmService.importBrAPIGermplasm(List.of(germplasm), new ArrayList<>(), program.getId(), null)); Flowable> enableCallRemove = client.exchange( PUT(enableUrl, List.of()).cookie(new NettyCookie("phylo-token", "test-registered-user")) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 9b8db2598..bef84eaf2 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -895,7 +895,7 @@ private Program createProgram(String name, String abbv, String key, String refer germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); - germplasmDAO.importBrAPIGermplasm(germplasm, program.getId(), null); + germplasmDAO.importBrAPIGermplasm(germplasm, new ArrayList<>(), program.getId(), null); } if(traits != null && !traits.isEmpty()) { From 7172d7a24e03f0dd90371f5dbc3ab687b7f92b01 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Fri, 17 Mar 2023 14:42:02 -0400 Subject: [PATCH 12/23] Fix migration version merge conflict --- ...m_import_add_GID.sql => V1.0.14__germplasm_import_add_GID.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V1.0.13__germplasm_import_add_GID.sql => V1.0.14__germplasm_import_add_GID.sql} (100%) diff --git a/src/main/resources/db/migration/V1.0.13__germplasm_import_add_GID.sql b/src/main/resources/db/migration/V1.0.14__germplasm_import_add_GID.sql similarity index 100% rename from src/main/resources/db/migration/V1.0.13__germplasm_import_add_GID.sql rename to src/main/resources/db/migration/V1.0.14__germplasm_import_add_GID.sql From de17fde3b4ef30969f8cc4871c691f51fdfd340d Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Mon, 27 Mar 2023 11:32:49 -0400 Subject: [PATCH 13/23] Fix file causing test failure --- .../files/germplasm_import/no_female_parent_blank_pedigree.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/resources/files/germplasm_import/no_female_parent_blank_pedigree.csv b/src/test/resources/files/germplasm_import/no_female_parent_blank_pedigree.csv index 9914816a4..8d1a4b461 100644 --- a/src/test/resources/files/germplasm_import/no_female_parent_blank_pedigree.csv +++ b/src/test/resources/files/germplasm_import/no_female_parent_blank_pedigree.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,BCR,Wild,0,2,,,,1234, ,Germplasm 2,BCR,Wild,0,3,,,,5678, ,Germplasm 3,BCR,Wild,0,3,,,,9123, \ No newline at end of file From 3560fbafcf27d05c1e35157a996f69d860df267b Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Tue, 28 Mar 2023 14:35:24 -0400 Subject: [PATCH 14/23] Fix list issue and cleanup --- .../brapps/importer/model/base/Germplasm.java | 5 +++++ .../services/processors/GermplasmProcessor.java | 10 ---------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java index b68180a13..da898722f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java @@ -189,6 +189,11 @@ public void updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID public void setUpdateCommitFields(BrAPIGermplasm germplasm, String programKey) { + + // Set germplasm name to [-] + String name = Utilities.appendProgramKey(germplasm.getDefaultDisplayName(), programKey, germplasm.getAccessionNumber()); + germplasm.setGermplasmName(name); + // Update our synonyms to [-] if (germplasm.getSynonyms() != null && !germplasm.getSynonyms().isEmpty()) { for (BrAPIGermplasmSynonyms synonym: germplasm.getSynonyms()) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index a75139322..8ac6331de 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -376,10 +376,6 @@ public Map process(List importRows return getStatisticsMap(importRows); } - private void processUpdatedGermplasm() { - - } - private Map getStatisticsMap(List importRows) { ImportPreviewStatistics germplasmStats = ImportPreviewStatistics.builder() @@ -387,12 +383,6 @@ private Map getStatisticsMap(List .ignoredObjectCount(germplasmByAccessionNumber.size()) .build(); - /* - .newObjectCount(ProcessorData.getNumNewObjects(observationByHash)) - .ignoredObjectCount(ProcessorData.getNumExistingObjects(observationByHash)) - */ - - //Modified logic here to check for female parent dbid or entry no, removed check for male due to assumption that shouldn't have only male parent int newObjectCount = newGermplasmList.stream().filter(newGermplasm -> newGermplasm != null).collect(Collectors.toList()).size(); ImportPreviewStatistics pedigreeConnectStats = ImportPreviewStatistics.builder() From de75621bb2986477b594c50156003f7c5f50aa6a Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Tue, 28 Mar 2023 16:33:02 -0400 Subject: [PATCH 15/23] Remove extraneous stuff from token controller --- .../breedinginsight/api/v1/controller/TokenController.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java b/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java index 4823e65d0..523292ac7 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java @@ -26,7 +26,6 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.auth.SecurityService; -import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.ApiToken; import org.breedinginsight.services.TokenService; @@ -42,8 +41,6 @@ public class TokenController { private SecurityService securityService; private TokenService tokenService; - private UserDAO userDAO; - @Inject public TokenController(SecurityService securityService, TokenService tokenService) { this.securityService = securityService; @@ -73,5 +70,4 @@ public HttpResponse apiToken(@QueryValue @NotBlank String returnUrl) { } } - } \ No newline at end of file From dcb12ca9878a10f9838caf123bcb2114b632f374 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Tue, 28 Mar 2023 16:38:30 -0400 Subject: [PATCH 16/23] cleanup --- .../org/breedinginsight/api/v1/controller/TokenController.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java b/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java index 523292ac7..ab68cb92f 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java @@ -69,5 +69,6 @@ public HttpResponse apiToken(@QueryValue @NotBlank String returnUrl) { return HttpResponse.serverError(); } + } -} \ No newline at end of file +} From 690cf3b210b950cb8281e21fd82610f14700d803 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Tue, 28 Mar 2023 16:40:21 -0400 Subject: [PATCH 17/23] spacing --- .../org/breedinginsight/api/v1/controller/TokenController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java b/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java index ab68cb92f..6c5442585 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java @@ -69,6 +69,6 @@ public HttpResponse apiToken(@QueryValue @NotBlank String returnUrl) { return HttpResponse.serverError(); } - } + } From 10f50f007a31ff83516b34ea27af3196f251dff0 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Fri, 7 Apr 2023 14:57:24 -0400 Subject: [PATCH 18/23] Code review changes work in progress --- .../brapi/v2/dao/BrAPIGermplasmDAO.java | 20 ++++++++++++++----- .../v2/services/BrAPIGermplasmService.java | 10 +++++++++- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index eb45c3cd3..14ffdf6b3 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -25,13 +25,11 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.client.v2.model.queryParams.germplasm.GermplasmQueryParams; import org.brapi.client.v2.modules.germplasm.GermplasmApi; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.germ.BrAPIGermplasmSynonyms; import org.brapi.v2.model.germ.request.BrAPIGermplasmSearchRequest; -import org.brapi.v2.model.germ.response.BrAPIGermplasmListResponse; import org.brapi.v2.model.germ.response.BrAPIGermplasmSingleResponse; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.daos.ImportDAO; @@ -240,17 +238,29 @@ private Map processGermplasmForDisplay(List importBrAPIGermplasm(List postBrAPIGermplasmList, List putBrAPIGermplasmList, UUID programId, ImportUpload upload) throws ApiException { + public List createBrAPIGermplasm(List postBrAPIGermplasmList, UUID programId, ImportUpload upload) { GermplasmApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), GermplasmApi.class); var program = programDAO.fetchOneById(programId); - Callable> postFunction; + Callable> postFunction = null; try { if (!postBrAPIGermplasmList.isEmpty()) { postFunction = () -> { List postResponse = brAPIDAOUtil.post(postBrAPIGermplasmList, upload, api::germplasmPost, importDAO::update); return processGermplasmForDisplay(postResponse, program.getKey()); }; - } else { + } + return programGermplasmCache.post(programId, postFunction); + } catch (Exception e) { + throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); + } + } + + public List updateBrAPIGermplasm(List putBrAPIGermplasmList, UUID programId, ImportUpload upload) { + GermplasmApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), GermplasmApi.class); + var program = programDAO.fetchOneById(programId); + Callable> postFunction = null; + try { + if (!putBrAPIGermplasmList.isEmpty()) { postFunction = () -> { List postResponse = putGermplasm(putBrAPIGermplasmList, api); return processGermplasmForDisplay(postResponse, program.getKey()); diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java index 7a9c523e6..527ad1272 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java @@ -285,9 +285,17 @@ private String removeAppendedKey(String listName, String programKey){ return listName.replace(appendedKey, ""); } + public List createBrAPIGermplasm(List postBrAPIGermplasmList, UUID programId, ImportUpload upload) { + return germplasmDAO.createBrAPIGermplasm(postBrAPIGermplasmList, programId, upload); + } + + public List updateBrAPIGermplasm(List putBrAPIGermplasmList, UUID programId, ImportUpload upload) { + return germplasmDAO.updateBrAPIGermplasm(putBrAPIGermplasmList, programId, upload); + } + public List importBrAPIGermplasm(List postBrAPIGermplasmList, List putBrAPIGermplasmList, UUID programId, ImportUpload upload) throws ApiException { - return germplasmDAO.importBrAPIGermplasm(postBrAPIGermplasmList, putBrAPIGermplasmList, programId, upload); + return germplasmDAO.createBrAPIGermplasm(postBrAPIGermplasmList, programId, upload); } public List getRawGermplasmByAccessionNumber(ArrayList germplasmAccessionNumbers, UUID programId) throws ApiException { From d8ebfc2d471d66a21b2bae0e7e7a54cea59c8eba Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Fri, 7 Apr 2023 14:59:52 -0400 Subject: [PATCH 19/23] Updated test --- .../brapps/importer/ExperimentFileImportTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index bef84eaf2..3f8df5c6c 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -895,7 +895,7 @@ private Program createProgram(String name, String abbv, String key, String refer germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); - germplasmDAO.importBrAPIGermplasm(germplasm, new ArrayList<>(), program.getId(), null); + germplasmDAO.createBrAPIGermplasm(germplasm, program.getId(), null); } if(traits != null && !traits.isEmpty()) { From 3cee4ee43757105737730abcd1fd6664cd69f938 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Mon, 10 Apr 2023 15:14:10 -0400 Subject: [PATCH 20/23] Split out method --- .../services/processors/GermplasmProcessor.java | 14 +++----------- .../BreedingMethodControllerIntegrationTest.java | 6 +++--- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index 8ac6331de..fc860c158 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -470,22 +470,14 @@ public void postBrapiData(Map mappedBrAPIImport, Program List createdGermplasm = new ArrayList<>(); if (!newGermplasmList.isEmpty()) { - try { - for (List postGroup: postOrder){ - createdGermplasm.addAll(brAPIGermplasmService.importBrAPIGermplasm(postGroup, new ArrayList<>(), program.getId(), upload)); - } - } catch (ApiException e) { - throw new InternalServerException(e.toString(), e); + for (List postGroup: postOrder){ + createdGermplasm.addAll(brAPIGermplasmService.createBrAPIGermplasm(postGroup, program.getId(), upload)); } } // PUT germplasm if (!updatedGermplasmList.isEmpty()) { - try { - brAPIGermplasmService.importBrAPIGermplasm(new ArrayList<>(), updatedGermplasmList, program.getId(), upload); - } catch (ApiException e) { - throw new InternalServerException(e.toString(), e); - } + brAPIGermplasmService.updateBrAPIGermplasm(updatedGermplasmList, program.getId(), upload); } // Create list diff --git a/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java index 8336a8d16..ed84a3fa2 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java @@ -400,7 +400,7 @@ public void createGermplasmWithProgramMethod() throws ApiException { .externalReferences(List.of(programRef, germIdRef)) .accessionNumber(accessionNum); - assertDoesNotThrow(() -> germplasmService.importBrAPIGermplasm(List.of(germplasm), new ArrayList<>(), program.getId(), null)); + assertDoesNotThrow(() -> germplasmService.createBrAPIGermplasm(List.of(germplasm), program.getId(), null)); String germplasmUrl = String.format("/programs/%s/brapi/v2/germplasm", program.getId()); Flowable> germplasmCall = client.exchange( @@ -472,7 +472,7 @@ public void tryDeleteProgramMethodInUse() throws ApiException { .externalReferences(List.of(programRef, germIdRef)) .accessionNumber(accessionNum); - assertDoesNotThrow(() -> germplasmService.importBrAPIGermplasm(List.of(germplasm), new ArrayList<>(), program.getId(), null)); + assertDoesNotThrow(() -> germplasmService.createBrAPIGermplasm(List.of(germplasm), program.getId(), null)); String deleteUrl = String.format("/programs/%s/breeding-methods/%s", program.getId(), createdMethod.getId()); Flowable> deleteCall = client.exchange( @@ -548,7 +548,7 @@ public void tryDisableSystemMethodInUse() { .externalReferences(List.of(programRef, germIdRef)) .accessionNumber(accessionNum); - assertDoesNotThrow(() -> germplasmService.importBrAPIGermplasm(List.of(germplasm), new ArrayList<>(), program.getId(), null)); + assertDoesNotThrow(() -> germplasmService.createBrAPIGermplasm(List.of(germplasm), program.getId(), null)); Flowable> enableCallRemove = client.exchange( PUT(enableUrl, List.of()).cookie(new NettyCookie("phylo-token", "test-registered-user")) From c16d20760baaf78c69bdfbf517563bbb0b922590 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Mon, 10 Apr 2023 16:32:29 -0400 Subject: [PATCH 21/23] Removed comment --- .../importer/services/processors/GermplasmProcessor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index fc860c158..667b56905 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -242,8 +242,7 @@ public Map process(List importRows Germplasm germplasm = importRows.get(0).getGermplasm(); importList = germplasm.constructBrAPIList(program, BRAPI_REFERENCE_SOURCE); } - - // All rows are considered new germplasm, we don't check for duplicates //todo except now we do oops + newGermplasmList = new ArrayList<>(); updatedGermplasmList = new ArrayList<>(); Map breedingMethods = new HashMap<>(); From 5d5d4143fb2fc2734c6be73e1e6791f518ef405e Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Tue, 11 Apr 2023 10:09:44 -0400 Subject: [PATCH 22/23] Added unsupported operation exception --- .../importer/services/processors/GermplasmProcessor.java | 2 +- src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index 667b56905..7f43cad42 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -242,7 +242,7 @@ public Map process(List importRows Germplasm germplasm = importRows.get(0).getGermplasm(); importList = germplasm.constructBrAPIList(program, BRAPI_REFERENCE_SOURCE); } - + newGermplasmList = new ArrayList<>(); updatedGermplasmList = new ArrayList<>(); Map breedingMethods = new HashMap<>(); diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index 6d8f495e9..3abb9dcc8 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -281,8 +281,7 @@ public List put(List brapiObjects, ImportUpload upload, Function, ApiResponse> putMethod, Consumer progressUpdateMethod) throws ApiException { - - return new ArrayList<>(); + throw new UnsupportedOperationException(); } public List post(List brapiObjects, From e43f6221f8f404c6a52b0211a1696551dc11d263 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Fri, 14 Apr 2023 11:26:28 -0400 Subject: [PATCH 23/23] Pedigree logic updated --- .../brapps/importer/model/base/Germplasm.java | 16 ++++++++++++++++ .../services/processors/GermplasmProcessor.java | 12 +++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java index da898722f..725bb9e44 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java @@ -17,6 +17,7 @@ package org.breedinginsight.brapps.importer.model.base; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; import lombok.Getter; import lombok.NoArgsConstructor; @@ -202,6 +203,21 @@ public void setUpdateCommitFields(BrAPIGermplasm germplasm, String programKey) { } } + public boolean pedigreesEqual(BrAPIGermplasm brAPIGermplasm) { + JsonElement femaleGid = brAPIGermplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_GID); + String brapiFemaleGid = femaleGid != null ? femaleGid.getAsString() : null; + JsonElement maleGid = brAPIGermplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID); + String brapiMaleGid = maleGid != null ? maleGid.getAsString() : null; + JsonElement femaleEntryNo = brAPIGermplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_ENTRY_NO); + String brapiFemaleEntryNo = femaleEntryNo != null ? femaleEntryNo.getAsString() : null; + JsonElement maleEntryNo = brAPIGermplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_ENTRY_NO); + String brapiMaleEntryNo = maleEntryNo != null ? maleEntryNo.getAsString() : null; + + return ((getFemaleParentDBID() == null && brapiFemaleGid == null) || (getFemaleParentDBID() != null && getFemaleParentDBID().equals(brapiFemaleGid))) && + ((getMaleParentDBID() == null && brapiMaleGid == null) || (getMaleParentDBID() != null && getMaleParentDBID().equals(brapiMaleGid))) && + ((getFemaleParentEntryNo() == null && brapiFemaleEntryNo == null) || (getFemaleParentEntryNo() != null && getFemaleParentEntryNo().equals(brapiFemaleEntryNo))) && + ((getMaleParentEntryNo() == null && brapiMaleEntryNo == null) || (getMaleParentEntryNo() != null && getMaleParentEntryNo().equals(brapiMaleEntryNo))); + } public BrAPIGermplasm constructBrAPIGermplasm(ProgramBreedingMethodEntity breedingMethod, User user, UUID listId) { BrAPIGermplasm germplasm = new BrAPIGermplasm(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index 7f43cad42..097f59fb1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -287,11 +287,13 @@ public Map process(List importRows continue; } - // Has an existing pedigree so add error and skip - // no pedigree on second import, don't change existing - // if there is pedigree & previously not, upgrade pedigree - // if previous pedigree & different pedigree, validation error - if (!StringUtils.isBlank(existingGermplasm.getPedigree())) { + // Error conditions: + // has existing pedigree and file pedigree is different + // Valid conditions: + // no existing pedigree and file different pedigree (not blank though, will fail other validations) + // existing pedigree and file pedigree same + + if (!StringUtils.isBlank(existingGermplasm.getPedigree()) && !germplasm.pedigreesEqual(existingGermplasm) ) { ValidationError ve = new ValidationError("Pedigree", pedigreeAlreadyExists, HttpStatus.UNPROCESSABLE_ENTITY); validationErrors.addError(i+2, ve ); // +2 instead of +1 to account for the column header row. continue;