From a542e7638acb79019429acf04e0ceb1b8d8655d6 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 11 Dec 2023 10:52:31 -0500 Subject: [PATCH 01/10] Added migration for new experiment sub unit fields --- .../V1.18.0__update_experiment_sub_unit.sql | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/main/resources/db/migration/V1.18.0__update_experiment_sub_unit.sql diff --git a/src/main/resources/db/migration/V1.18.0__update_experiment_sub_unit.sql b/src/main/resources/db/migration/V1.18.0__update_experiment_sub_unit.sql new file mode 100644 index 000000000..6469fd0f6 --- /dev/null +++ b/src/main/resources/db/migration/V1.18.0__update_experiment_sub_unit.sql @@ -0,0 +1,26 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +DO $$ + +BEGIN +update importer_mapping +set + mapping = '[{"id": "726a9f10-4892-4204-9e52-bd2b1d735f65", "value": {"fileFieldName": "Germplasm Name"}, "objectId": "germplasmName"}, {"id": "98774e20-6f89-4d6a-a7c9-f88887228ed6", "value": {"fileFieldName": "Germplasm GID"}, "objectId": "gid"}, {"id": "880ef0c9-4e3e-42d4-9edc-667684a91889", "value": {"fileFieldName": "Test (T) or Check (C)"}, "objectId": "test_or_check"}, {"id": "b693eca7-efcd-4518-a9d3-db0b037a76ee", "value": {"fileFieldName": "Exp Title"}, "objectId": "exp_title"}, {"id": "df340215-db6e-4219-a3b7-119f297b81c3", "value": {"fileFieldName": "Exp Description"}, "objectId": "expDescription"}, {"id": "9ca7cc81-562c-43a7-989a-41da309f603d", "value": {"fileFieldName": "Exp Unit"}, "objectId": "expUnit"}, {"id": "1b0ffa01-c215-4b94-9211-b8ccb05693d4", "value": {"fileFieldName": "Sub-Obs Unit"}, "objectId": "subObsUnit"}, {"id": "27215777-c8f9-4fe7-a7ac-918d6168b0dd", "value": {"fileFieldName": "Exp Type"}, "objectId": "expType"}, {"id": "19d220e2-dff0-4a3a-ad6e-32f4d8602b5c", "value": {"fileFieldName": "Env"}, "objectId": "env"}, {"id": "861518b9-5c9e-4fe5-b31e-baf16e27155d", "value": {"fileFieldName": "Env Location"}, "objectId": "envLocation"}, {"id": "667355c3-dae1-4a64-94c8-ac2d543bd474", "value": {"fileFieldName": "Env Year"}, "objectId": "envYear"}, {"id": "ad11f2df-c5b4-4a05-8e52-c57625140061", "value": {"fileFieldName": "Exp Unit ID"}, "objectId": "expUnitId"}, {"id": "709d69e3-7f62-463b-8571-a3aed18f7801", "value": {"fileFieldName": "Sub Unit ID"}, "objectId": "subUnitId"}, {"id": "639b40ec-20f8-4659-8464-6a4be997ac7a", "value": {"fileFieldName": "Exp Replicate #"}, "objectId": "expReplicateNo"}, {"id": "2a62a80f-d8ba-42c4-9997-3b4ac8a965aa", "value": {"fileFieldName": "Exp Block #"}, "objectId": "expBlockNo"}, {"id": "f3e7de69-21ad-4cda-b1cc-a5e1987fb931", "value": {"fileFieldName": "Row"}, "objectId": "row"}, {"id": "251c5bbd-fc4d-4371-a4ce-4e2686f6837e", "value": {"fileFieldName": "Column"}, "objectId": "column"}, {"id": "a0edcc27-d423-478f-8eed-05b554805ec9", "value": {"fileFieldName": "Lat"}, "objectId": "lat"}, {"id": "a8fd2190-d277-4369-ae72-af32416f14ac", "value": {"fileFieldName": "Long"}, "objectId": "long"}, {"id": "95caefec-2b12-4728-9f2e-3bc17478b662", "value": {"fileFieldName": "Elevation"}, "objectId": "elevation"}, {"id": "e8f80336-0982-4a48-85ac-4b0278e28b70", "value": {"fileFieldName": "RTK"}, "objectId": "rtk"}, {"id": "ce5f61f2-f1de-45a4-8baf-e2471a5d863d", "value": {"fileFieldName": "Treatment Factors"}, "objectId": "treatmentFactors"}, {"id": "c5b8276f-e777-4385-a80f-5199abe63aac", "value": {"fileFieldName": "ObsUnitID"}, "objectId": "ObsUnitID"}]', + file = '[{"Germplasm Name": "Test", "Germplasm GID": 1, "Test (T) or Check (C)": true, "Exp Title": "New Trial", "Exp Description": "A new trial", "Exp Unit": "plot", "Sub-Obs Unit": "plant", "Exp Type": "phenotyping", "Env": "New Study", "Env Location": "New Location", "Env Year": 2012, "Exp Unit ID": 3, "Sub Unit ID": 1, "Exp Replicate #": 0, "Exp Block #": 2, "Row": 4, "Column": 5, "Lat": 42.4440, "Long": 76.5019, "Elevation": 123, "RTK": "RTK description", "Treatment Factors": "Jam application", "ObsUnitID": ""}]' +where import_type_id = 'ExperimentImport'; +END $$; \ No newline at end of file From 5680414168642fdaefd0cfaed5d4482fb3b72ca5 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 11 Dec 2023 17:09:09 -0500 Subject: [PATCH 02/10] Added validations for geocoordinates --- .../ExperimentObservation.java | 55 ++++++++++++++ .../processors/ExperimentProcessor.java | 75 +++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index f45cd83a3..ca4e44aec 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -17,10 +17,14 @@ package org.breedinginsight.brapps.importer.model.imports.experimentObservation; +import com.github.filosganga.geogson.model.Feature; +import com.github.filosganga.geogson.model.Point; +import com.github.filosganga.geogson.model.positions.SinglePosition; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.BrApiGeoJSON; import org.brapi.v2.model.core.*; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.*; @@ -72,6 +76,10 @@ public class ExperimentObservation implements BrAPIImport { @ImportFieldMetadata(id = "expUnit", name = Columns.EXP_UNIT, description = "Experiment unit (Examples: plots, plant, tanks, hives, etc.)") private String expUnit; + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "subObsUnit", name = Columns.SUB_OBS_UNIT, description = "Sub observation unit (Examples: plant, etc.)") + private String subObsUnit; + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) @ImportFieldMetadata(id = "expType", name = Columns.EXP_TYPE, description = "Description of experimental type (Examples: Performance trial, crossing block, seed orchard, etc)") private String expType; @@ -92,6 +100,10 @@ public class ExperimentObservation implements BrAPIImport { @ImportFieldMetadata(id = "expUnitId", name = Columns.EXP_UNIT_ID, description = "Human-readable alphanumeric identifier for experimental units unique within environment. Examples, like plot number, are often a numeric sequence.") private String expUnitId; + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "subUnitId", name = Columns.SUB_UNIT_ID, description = "Alphanumeric identifier of sub-units. For example if three fruits are observed per plot, the three fruits can be identified by 1,2,3.") + private String subUnitId; + @ImportFieldType(type = ImportFieldTypeEnum.INTEGER) @ImportFieldMetadata(id = "expReplicateNo", name = Columns.REP_NUM, description = "Sequential number of experimental replications") private String expReplicateNo; @@ -108,6 +120,22 @@ public class ExperimentObservation implements BrAPIImport { @ImportFieldMetadata(id = "column", name = Columns.COLUMN, description = "Vertical (x-axis) position in 2D Cartesian space.") private String column; + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "lat", name = Columns.LAT, description = "Latitude coordinate in WGS 84 coordinate reference system.") + private String latitude; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "long", name = Columns.LONG, description = "Longitude coordinate in WGS 84 coordinate reference system.") + private String longitude; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "elevation", name = Columns.ELEVATION, description = "Height in meters above WGS 84 reference ellipsoid.") + private String elevation; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "rtk", name = Columns.RTK, description = "Free text description of real-time kinematic positioning used to correct coordinates.") + private String rtk; + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) @ImportFieldMetadata(id = "treatmentFactors", name = Columns.TREATMENT_FACTORS, description = "Treatment factors in an experiment with applied variables, like fertilizer or water regimens.") private String treatmentFactors; @@ -279,6 +307,27 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( position.setEntryType(BrAPIEntryTypeEnum.TEST); } + // geocoordinates + try { + double lat = Double.parseDouble(getLatitude()); + double lon = Double.parseDouble(getLongitude()); + Point geoPoint = Point.from(lat, lon); + + if (getElevation() != null) { + double elevation = Double.parseDouble(getElevation()); + geoPoint = geoPoint.withAlt(elevation); + } + + BrApiGeoJSON coords = BrApiGeoJSON.builder() + .geometry(geoPoint) + .type("Feature") + .build(); + position.setGeoCoordinates(coords); + + } catch (NullPointerException | NumberFormatException e) { + // ignore null or number format exceptions, won't populate geocoordinates if there are any issues + } + // X and Y coordinates if (getRow() != null) { position.setPositionCoordinateX(getRow()); @@ -404,15 +453,21 @@ public static final class Columns { public static final String EXP_TITLE = "Exp Title"; public static final String EXP_DESCRIPTION = "Exp Description"; public static final String EXP_UNIT = "Exp Unit"; + public static final String SUB_OBS_UNIT = "Sub-Obs Unit"; public static final String EXP_TYPE = "Exp Type"; public static final String ENV = "Env"; public static final String ENV_LOCATION = "Env Location"; public static final String ENV_YEAR = "Env Year"; public static final String EXP_UNIT_ID = "Exp Unit ID"; + public static final String SUB_UNIT_ID = "Sub Unit ID"; public static final String REP_NUM = "Exp Replicate #"; public static final String BLOCK_NUM = "Exp Block #"; public static final String ROW = "Row"; public static final String COLUMN = "Column"; + public static final String LAT = "Lat"; + public static final String LONG = "Long"; + public static final String ELEVATION = "Elevation"; + public static final String RTK = "RTK"; public static final String TREATMENT_FACTORS = "Treatment Factors"; public static final String OBS_UNIT_ID = "ObsUnitID"; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index e5ec4a5c5..b853a36e5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -16,6 +16,9 @@ */ package org.breedinginsight.brapps.importer.services.processors; +import com.github.filosganga.geogson.model.Feature; +import com.github.filosganga.geogson.model.Point; +import com.github.filosganga.geogson.model.positions.SinglePosition; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; @@ -29,6 +32,7 @@ import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.BrApiGeoJSON; import org.brapi.v2.model.core.*; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; @@ -572,6 +576,7 @@ private ValidationErrors validateFields(List importRows, Validation validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + validateGeoCoordinates(validationErrors, rowNum, importRow); validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, existingObsByObsHash); } return validationErrors; @@ -587,6 +592,76 @@ private void validateObservationUnits(ValidationErrors validationErrors, Set 90) { + latBadValue = true; + } + } catch (NumberFormatException e) { + latBadValue = true; + } + } + + // Only check longitude format if not blank since already had previous error + if (StringUtils.isNotBlank(lon)) { + try { + lonDouble = Double.parseDouble(lon); + if (lonDouble < -180 || lonDouble > 180) { + lonBadValue = true; + } + } catch (NumberFormatException e) { + lonBadValue = true; + } + } + + if (StringUtils.isNotBlank(elevation)) { + try { + elevationDouble = Double.parseDouble(elevation); + } catch (NumberFormatException e) { + elevationBadValue = true; + } + } + + if (latBadValue) { + addRowError(Columns.LAT, "Invalid Lat value (expected range -90 to 90)", validationErrors, rowNum); + } + + if (lonBadValue) { + addRowError(Columns.LONG, "Invalid Long value (expected range -180 to 180)", validationErrors, rowNum); + } + + if (elevationBadValue) { + addRowError(Columns.LONG, "Invalid Elevation value (numerals expected)", validationErrors, rowNum); + } + + } + private Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { Set ouDbIds = new HashSet<>(); Set variableDbIds = new HashSet<>(); From 07739d29dfc2ca77365bfa561450c3014a67e33f Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 11 Dec 2023 18:15:51 -0500 Subject: [PATCH 03/10] Removed unused imports --- .../importer/services/processors/ExperimentProcessor.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index b853a36e5..0323884b5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -16,10 +16,6 @@ */ package org.breedinginsight.brapps.importer.services.processors; -import com.github.filosganga.geogson.model.Feature; -import com.github.filosganga.geogson.model.Point; -import com.github.filosganga.geogson.model.positions.SinglePosition; -import com.google.gson.JsonElement; import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; @@ -32,7 +28,6 @@ import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.BrApiGeoJSON; import org.brapi.v2.model.core.*; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; @@ -77,7 +72,6 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.*; -import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; From 2ad3f90082b33e915a4a9cbfea86a07356247f55 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 11 Dec 2023 18:21:56 -0500 Subject: [PATCH 04/10] Moved geocoordinate validation to be called from observation unit validation --- .../importer/services/processors/ExperimentProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index bf63a7bc2..04c204c38 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -611,7 +611,6 @@ private void validateFields(List importRows, ValidationErrors valid validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); - validateGeoCoordinates(validationErrors, rowNum, importRow); validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); } @@ -625,6 +624,8 @@ private void validateObservationUnits(ValidationErrors validationErrors, Set Date: Tue, 12 Dec 2023 09:50:22 -0500 Subject: [PATCH 05/10] Added new columns to test file creation --- .../breedinginsight/brapps/importer/ImportTestUtils.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index 3a521aa55..a9063233b 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -206,15 +206,21 @@ public File writeExperimentDataToFile(List> data, List Date: Tue, 12 Dec 2023 10:14:17 -0500 Subject: [PATCH 06/10] Added columns for experiment controller tests --- .../brapi/v2/ExperimentControllerIntegrationTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index c3496ba41..5f3c044be 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -289,15 +289,21 @@ private Map makeExpImportRow(String environment) { row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); row.put(ExperimentObservation.Columns.EXP_TITLE, "Test Exp"); row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.SUB_OBS_UNIT, ""); row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); row.put(ExperimentObservation.Columns.ENV, environment); row.put(ExperimentObservation.Columns.ENV_LOCATION, "Location A"); row.put(ExperimentObservation.Columns.ENV_YEAR, "2023"); row.put(ExperimentObservation.Columns.EXP_UNIT_ID, "a-1"); + row.put(ExperimentObservation.Columns.SUB_UNIT_ID, ""); row.put(ExperimentObservation.Columns.REP_NUM, "1"); row.put(ExperimentObservation.Columns.BLOCK_NUM, "1"); row.put(ExperimentObservation.Columns.ROW, "1"); row.put(ExperimentObservation.Columns.COLUMN, "1"); + row.put(ExperimentObservation.Columns.LAT, ""); + row.put(ExperimentObservation.Columns.LONG, ""); + row.put(ExperimentObservation.Columns.ELEVATION, ""); + row.put(ExperimentObservation.Columns.RTK, ""); return row; } From 81378913c8a30e60253e719520cd02cf9ed6f955 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 12 Dec 2023 13:33:14 -0500 Subject: [PATCH 07/10] More test fixes --- .../services/parsers/experiment/ExperimentFileColumns.java | 6 ++++++ src/main/java/org/breedinginsight/utilities/FileUtil.java | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java index 5ddb65a76..f95283bf2 100644 --- a/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java +++ b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java @@ -32,15 +32,21 @@ public enum ExperimentFileColumns { EXP_TITLE(ExperimentObservation.Columns.EXP_TITLE, Column.ColumnDataType.STRING), EXP_DESCRIPTION(ExperimentObservation.Columns.EXP_DESCRIPTION, Column.ColumnDataType.STRING), EXP_UNIT(ExperimentObservation.Columns.EXP_UNIT, Column.ColumnDataType.STRING), + SUB_OBS_UNIT(ExperimentObservation.Columns.SUB_OBS_UNIT, Column.ColumnDataType.STRING), EXP_TYPE(ExperimentObservation.Columns.EXP_TYPE, Column.ColumnDataType.STRING), ENV(ExperimentObservation.Columns.ENV, Column.ColumnDataType.STRING), ENV_LOCATION(ExperimentObservation.Columns.ENV_LOCATION, Column.ColumnDataType.STRING), ENV_YEAR(ExperimentObservation.Columns.ENV_YEAR, Column.ColumnDataType.INTEGER), EXP_UNIT_ID(ExperimentObservation.Columns.EXP_UNIT_ID, Column.ColumnDataType.STRING), + SUB_UNIT_ID(ExperimentObservation.Columns.SUB_UNIT_ID, Column.ColumnDataType.STRING), REP_NUM(ExperimentObservation.Columns.REP_NUM, Column.ColumnDataType.INTEGER), BLOCK_NUM(ExperimentObservation.Columns.BLOCK_NUM, Column.ColumnDataType.INTEGER), ROW(ExperimentObservation.Columns.ROW, Column.ColumnDataType.STRING), COLUMN(ExperimentObservation.Columns.COLUMN, Column.ColumnDataType.STRING), + LAT(ExperimentObservation.Columns.LAT, Column.ColumnDataType.STRING), + LONG(ExperimentObservation.Columns.LONG, Column.ColumnDataType.STRING), + ELEVATION(ExperimentObservation.Columns.ELEVATION, Column.ColumnDataType.STRING), + RTK(ExperimentObservation.Columns.RTK, Column.ColumnDataType.STRING), TREATMENT_FACTORS(ExperimentObservation.Columns.TREATMENT_FACTORS, Column.ColumnDataType.STRING), OBS_UNIT_ID(ExperimentObservation.Columns.OBS_UNIT_ID, Column.ColumnDataType.STRING); diff --git a/src/main/java/org/breedinginsight/utilities/FileUtil.java b/src/main/java/org/breedinginsight/utilities/FileUtil.java index 985103bc1..e5703d9e8 100644 --- a/src/main/java/org/breedinginsight/utilities/FileUtil.java +++ b/src/main/java/org/breedinginsight/utilities/FileUtil.java @@ -173,7 +173,9 @@ public static Table parseTableFromCsv(InputStream inputStream) throws ParsingExc // Convert to Table. //Jackson used downstream messily converts LOCAL_DATE/LOCAL_DATETIME, so need to interpret date input as strings //Note that if another type is needed later this is what needs to be updated - ArrayList acceptedTypes = new ArrayList<>(Arrays.asList(ColumnType.STRING, ColumnType.INTEGER, ColumnType.DOUBLE, ColumnType.FLOAT)); + //Removed FLOAT and DOUBLE types because it was causing issues with experiment geocoordinates which are to be treated as strings + //until validations are done + ArrayList acceptedTypes = new ArrayList<>(Arrays.asList(ColumnType.STRING, ColumnType.INTEGER)); Table df = Table.read().usingOptions( CsvReadOptions .builderFromString(input) From ce54b7e4c0180d515027cc8bdad60b9e94d01b35 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:18:16 -0500 Subject: [PATCH 08/10] Fix elevation not being added to geocoordinates --- .../imports/experimentObservation/ExperimentObservation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 79f60648b..9fd18712a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -322,7 +322,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( if (getElevation() != null) { double elevation = Double.parseDouble(getElevation()); - geoPoint = geoPoint.withAlt(elevation); + geoPoint = Point.from(lat, lon, elevation); // geoPoint.withAlt(elevation) did not work } BrApiGeoJSON coords = BrApiGeoJSON.builder() From 8003d1327eac1176ec0e6f2047c5381c642811c9 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 2 Jan 2024 11:58:01 -0500 Subject: [PATCH 09/10] Added rtk to additionalinfo --- .../brapi/v2/constants/BrAPIAdditionalInfoFields.java | 1 + .../experimentObservation/ExperimentObservation.java | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 27723e9d5..87fbfdda9 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -38,6 +38,7 @@ public final class BrAPIAdditionalInfoFields { public static final String CREATED_DATE = "createdDate"; public static final String DEFAULT_OBSERVATION_LEVEL = "defaultObservationLevel"; public static final String OBSERVATION_LEVEL = "observationLevel"; + public static final String RTK = "rtk"; public static final String EXPERIMENT_TYPE = "experimentType"; public static final String EXPERIMENT_NUMBER = "experimentNumber"; public static final String ENVIRONMENT_NUMBER = "environmentNumber"; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 9fd18712a..a2b139bab 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -23,6 +23,7 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; +import org.apache.commons.lang3.StringUtils; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.BrApiGeoJSON; import org.brapi.v2.model.core.*; @@ -335,6 +336,11 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( // ignore null or number format exceptions, won't populate geocoordinates if there are any issues } + String rtk = getRtk(); + if (StringUtils.isNotBlank(rtk)) { + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.RTK, rtk); + } + // X and Y coordinates if (getRow() != null) { position.setPositionCoordinateX(getRow()); From 87b1d72956c458b60898084b88ccea14b596a5da Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 3 Jan 2024 11:46:43 -0500 Subject: [PATCH 10/10] Fix geojson serialization problem --- .../serializer/CustomObjectMapperFactory.java | 54 +++++++++++++++++++ .../api/serializer/GsonBasedSerializer.java | 44 +++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 src/main/java/org/breedinginsight/api/serializer/CustomObjectMapperFactory.java create mode 100644 src/main/java/org/breedinginsight/api/serializer/GsonBasedSerializer.java diff --git a/src/main/java/org/breedinginsight/api/serializer/CustomObjectMapperFactory.java b/src/main/java/org/breedinginsight/api/serializer/CustomObjectMapperFactory.java new file mode 100644 index 000000000..32f018d96 --- /dev/null +++ b/src/main/java/org/breedinginsight/api/serializer/CustomObjectMapperFactory.java @@ -0,0 +1,54 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.api.serializer; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.gson.Gson; +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Replaces; +import io.micronaut.core.annotation.Nullable; +import io.micronaut.jackson.JacksonConfiguration; +import io.micronaut.jackson.ObjectMapperFactory; +import org.brapi.client.v2.JSON; +import org.brapi.v2.model.BrApiGeoJSON; + +import javax.inject.Singleton; + +/** + * Add custom serializers to Micronaut's Jackson ObjectMapper + */ +@Factory +public class CustomObjectMapperFactory extends ObjectMapperFactory { + + @Singleton @Replaces(ObjectMapper.class) + @Override + public ObjectMapper objectMapper(@Nullable JacksonConfiguration jacksonConfiguration, @Nullable JsonFactory jsonFactory) { + ObjectMapper mapper = super.objectMapper(jacksonConfiguration, jsonFactory); + + // Jackson was not properly serializing geojson objects from com.github.filosganga.geogson + // which is made to work with gson and part of the brapi client object models so just use gson to + // do the serialization instead of jackson for this case + SimpleModule module = new SimpleModule(); + Gson gson = new JSON().getGson(); + module.addSerializer(new GsonBasedSerializer<>(BrApiGeoJSON.class, gson)); + mapper.registerModule(module); + + return mapper; + } +} diff --git a/src/main/java/org/breedinginsight/api/serializer/GsonBasedSerializer.java b/src/main/java/org/breedinginsight/api/serializer/GsonBasedSerializer.java new file mode 100644 index 000000000..a1e10aedf --- /dev/null +++ b/src/main/java/org/breedinginsight/api/serializer/GsonBasedSerializer.java @@ -0,0 +1,44 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.serializer; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import com.google.gson.Gson; + +import java.io.IOException; + +/** + * Jackson Serializer that uses Gson to do the serialization + */ +public class GsonBasedSerializer extends StdSerializer { + + private final Gson gson; + + public GsonBasedSerializer(Class t, Gson gson) { + super(t); + this.gson = gson; + } + + @Override + public void serialize(T value, JsonGenerator gen, SerializerProvider provider) throws IOException { + String json = gson.toJson(value); + gen.writeRawValue(json); // Using writeRawValue to avoid double quoting + } +}