BI-2005 - Expand Exp Template

src/main/java/org/breedinginsight/api/serializer/CustomObjectMapperFactory.java

@@ -0,0 +1,54 @@
+/*
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.breedinginsight.api.serializer;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.gson.Gson;
+import io.micronaut.context.annotation.Factory;
+import io.micronaut.context.annotation.Replaces;
+import io.micronaut.core.annotation.Nullable;
+import io.micronaut.jackson.JacksonConfiguration;
+import io.micronaut.jackson.ObjectMapperFactory;
+import org.brapi.client.v2.JSON;
+import org.brapi.v2.model.BrApiGeoJSON;
+
+import javax.inject.Singleton;
+
+/**
+ * Add custom serializers to Micronaut's Jackson ObjectMapper
+ */
+@Factory
+public class CustomObjectMapperFactory extends ObjectMapperFactory {
+
+    @Singleton @Replaces(ObjectMapper.class)
+    @Override
+    public ObjectMapper objectMapper(@Nullable JacksonConfiguration jacksonConfiguration, @Nullable JsonFactory jsonFactory) {
+        ObjectMapper mapper = super.objectMapper(jacksonConfiguration, jsonFactory);
+
+        // Jackson was not properly serializing geojson objects from com.github.filosganga.geogson
+        // which is made to work with gson and part of the brapi client object models so just use gson to
+        // do the serialization instead of jackson for this case
+        SimpleModule module = new SimpleModule();
+        Gson gson = new JSON().getGson();
+        module.addSerializer(new GsonBasedSerializer<>(BrApiGeoJSON.class, gson));
+        mapper.registerModule(module);
+
+        return mapper;
+    }
+}

src/main/java/org/breedinginsight/api/serializer/GsonBasedSerializer.java

@@ -0,0 +1,44 @@
+/*
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.breedinginsight.api.serializer;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
+import com.google.gson.Gson;
+
+import java.io.IOException;
+
+/**
+ * Jackson Serializer that uses Gson to do the serialization
+ */
+public class GsonBasedSerializer<T> extends StdSerializer<T> {
+
+    private final Gson gson;
+
+    public GsonBasedSerializer(Class<T> t, Gson gson) {
+        super(t);
+        this.gson = gson;
+    }
+
+    @Override
+    public void serialize(T value, JsonGenerator gen, SerializerProvider provider) throws IOException {
+        String json = gson.toJson(value);
+        gen.writeRawValue(json); // Using writeRawValue to avoid double quoting
+    }
+}

src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java

@@ -38,6 +38,7 @@ public final class BrAPIAdditionalInfoFields {
     public static final String CREATED_DATE = "createdDate";
     public static final String DEFAULT_OBSERVATION_LEVEL = "defaultObservationLevel";
     public static final String OBSERVATION_LEVEL = "observationLevel";
+    public static final String RTK = "rtk";
     public static final String EXPERIMENT_TYPE = "experimentType";
     public static final String EXPERIMENT_NUMBER = "experimentNumber";
     public static final String ENVIRONMENT_NUMBER = "environmentNumber";

src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java

@@ -17,10 +17,15 @@
 
 package org.breedinginsight.brapps.importer.model.imports.experimentObservation;
 
+import com.github.filosganga.geogson.model.Feature;
+import com.github.filosganga.geogson.model.Point;
+import com.github.filosganga.geogson.model.positions.SinglePosition;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
 import org.brapi.v2.model.BrAPIExternalReference;
+import org.brapi.v2.model.BrApiGeoJSON;
 import org.brapi.v2.model.core.*;
 import org.brapi.v2.model.core.response.BrAPIListDetails;
 import org.brapi.v2.model.pheno.*;
@@ -79,6 +84,10 @@ public class ExperimentObservation implements BrAPIImport {
     @ImportFieldMetadata(id = "expUnit", name = Columns.EXP_UNIT, description = "Experiment unit  (Examples: plots, plant, tanks, hives, etc.)")
     private String expUnit;
 
+    @ImportFieldType(type = ImportFieldTypeEnum.TEXT)
+    @ImportFieldMetadata(id = "subObsUnit", name = Columns.SUB_OBS_UNIT, description = "Sub observation unit (Examples: plant, etc.)")
+    private String subObsUnit;
+
     @ImportFieldType(type = ImportFieldTypeEnum.TEXT)
     @ImportFieldMetadata(id = "expType", name = Columns.EXP_TYPE, description = "Description of experimental type (Examples: Performance trial, crossing block, seed orchard, etc)")
     private String expType;
@@ -99,6 +108,10 @@ public class ExperimentObservation implements BrAPIImport {
     @ImportFieldMetadata(id = "expUnitId", name = Columns.EXP_UNIT_ID, description = "Human-readable alphanumeric identifier for experimental units unique within environment. Examples, like plot number, are often a numeric sequence.")
     private String expUnitId;
 
+    @ImportFieldType(type = ImportFieldTypeEnum.TEXT)
+    @ImportFieldMetadata(id = "subUnitId", name = Columns.SUB_UNIT_ID, description = "Alphanumeric identifier of sub-units. For example if three fruits are observed per plot, the three fruits can be identified by 1,2,3.")
+    private String subUnitId;
+
     @ImportFieldType(type = ImportFieldTypeEnum.INTEGER)
     @ImportFieldMetadata(id = "expReplicateNo", name = Columns.REP_NUM, description = "Sequential number of experimental replications")
     private String expReplicateNo;
@@ -115,6 +128,22 @@ public class ExperimentObservation implements BrAPIImport {
     @ImportFieldMetadata(id = "column", name = Columns.COLUMN, description = "Vertical (x-axis) position in 2D Cartesian space.")
     private String column;
 
+    @ImportFieldType(type = ImportFieldTypeEnum.TEXT)
+    @ImportFieldMetadata(id = "lat", name = Columns.LAT, description = "Latitude coordinate in WGS 84 coordinate reference system.")
+    private String latitude;
+
+    @ImportFieldType(type = ImportFieldTypeEnum.TEXT)
+    @ImportFieldMetadata(id = "long", name = Columns.LONG, description = "Longitude coordinate in WGS 84 coordinate reference system.")
+    private String longitude;
+
+    @ImportFieldType(type = ImportFieldTypeEnum.TEXT)
+    @ImportFieldMetadata(id = "elevation", name = Columns.ELEVATION, description = "Height in meters above WGS 84 reference ellipsoid.")
+    private String elevation;
+
+    @ImportFieldType(type = ImportFieldTypeEnum.TEXT)
+    @ImportFieldMetadata(id = "rtk", name = Columns.RTK, description = "Free text description of real-time kinematic positioning used to correct coordinates.")
+    private String rtk;
+
     @ImportFieldType(type = ImportFieldTypeEnum.TEXT)
     @ImportFieldMetadata(id = "treatmentFactors", name = Columns.TREATMENT_FACTORS, description = "Treatment factors in an experiment with applied variables, like fertilizer or water regimens.")
     private String treatmentFactors;
@@ -288,6 +317,32 @@ public BrAPIObservationUnit constructBrAPIObservationUnit(
             position.setEntryType(BrAPIEntryTypeEnum.TEST);
         }
 
+        // geocoordinates
+        try {
+            double lat = Double.parseDouble(getLatitude());
+            double lon = Double.parseDouble(getLongitude());
+            Point geoPoint = Point.from(lat, lon);
+
+            if (getElevation() != null) {
+                double elevation = Double.parseDouble(getElevation());
+                geoPoint = Point.from(lat, lon, elevation); // geoPoint.withAlt(elevation) did not work
+            }
+
+            BrApiGeoJSON coords = BrApiGeoJSON.builder()
+                    .geometry(geoPoint)
+                    .type("Feature")
+                    .build();
+            position.setGeoCoordinates(coords);
+
+        } catch (NullPointerException | NumberFormatException e) {
+            // ignore null or number format exceptions, won't populate geocoordinates if there are any issues
+        }
+
+        String rtk = getRtk();
+        if (StringUtils.isNotBlank(rtk)) {
+            observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.RTK, rtk);
+        }
+
         // X and Y coordinates
         if (getRow() != null) {
             position.setPositionCoordinateX(getRow());
@@ -413,15 +468,21 @@ public static final class Columns {
         public static final String EXP_TITLE = "Exp Title";
         public static final String EXP_DESCRIPTION = "Exp Description";
         public static final String EXP_UNIT = "Exp Unit";
+        public static final String SUB_OBS_UNIT = "Sub-Obs Unit";
         public static final String EXP_TYPE = "Exp Type";
         public static final String ENV = "Env";
         public static final String ENV_LOCATION = "Env Location";
         public static final String ENV_YEAR = "Env Year";
         public static final String EXP_UNIT_ID = "Exp Unit ID";
+        public static final String SUB_UNIT_ID = "Sub Unit ID";
         public static final String REP_NUM = "Exp Replicate #";
         public static final String BLOCK_NUM = "Exp Block #";
         public static final String ROW = "Row";
         public static final String COLUMN = "Column";
+        public static final String LAT = "Lat";
+        public static final String LONG = "Long";
+        public static final String ELEVATION = "Elevation";
+        public static final String RTK = "RTK";
         public static final String TREATMENT_FACTORS = "Treatment Factors";
         public static final String OBS_UNIT_ID = "ObsUnitID";
     }

src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java

@@ -16,6 +16,7 @@
  */
 package org.breedinginsight.brapps.importer.services.processors;
 
+
 import com.google.gson.Gson;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonObject;
@@ -601,6 +602,7 @@ private void validateFields(List<BrAPIImport> importRows, ValidationErrors valid
             validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit);
             validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow);
             validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user);
+
         }
     }
 
@@ -612,6 +614,78 @@ private void validateObservationUnits(ValidationErrors validationErrors, Set<Str
         if(ouPIO.getState() == ImportObjectState.NEW && StringUtils.isNotBlank(importRow.getObsUnitID())) {
             addRowError(Columns.OBS_UNIT_ID, "Could not find observation unit by ObsUnitDBID", validationErrors, rowNum);
         }
+
+        validateGeoCoordinates(validationErrors, rowNum, importRow);
+    }
+
+    private void validateGeoCoordinates(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow) {
+
+        String lat = importRow.getLatitude();
+        String lon = importRow.getLongitude();
+        String elevation = importRow.getElevation();
+
+        // If any of Lat, Long, or Elevation are provided, Lat and Long must both be provided.
+        if (StringUtils.isNotBlank(lat) || StringUtils.isNotBlank(lon) || StringUtils.isNotBlank(elevation)) {
+            if (StringUtils.isBlank(lat)) {
+                addRowError(Columns.LAT, "Latitude must be provided for complete coordinate specification", validationErrors, rowNum);
+            }
+            if (StringUtils.isBlank(lon)) {
+                addRowError(Columns.LONG, "Longitude must be provided for complete coordinate specification", validationErrors, rowNum);
+            }
+        }
+
+        // Validate coordinate values
+        boolean latBadValue = false;
+        boolean lonBadValue = false;
+        boolean elevationBadValue = false;
+        double latDouble;
+        double lonDouble;
+        double elevationDouble;
+
+        // Only check latitude format if not blank since already had previous error
+        if (StringUtils.isNotBlank(lat)) {
+            try {
+                latDouble = Double.parseDouble(lat);
+                if (latDouble < -90 || latDouble > 90) {
+                    latBadValue = true;
+                }
+            } catch (NumberFormatException e) {
+                latBadValue = true;
+            }
+        }
+
+        // Only check longitude format if not blank since already had previous error
+        if (StringUtils.isNotBlank(lon)) {
+            try {
+                lonDouble = Double.parseDouble(lon);
+                if (lonDouble < -180 || lonDouble > 180) {
+                    lonBadValue = true;
+                }
+            } catch (NumberFormatException e) {
+                lonBadValue = true;
+            }
+        }
+
+        if (StringUtils.isNotBlank(elevation)) {
+            try {
+                elevationDouble = Double.parseDouble(elevation);
+            } catch (NumberFormatException e) {
+                elevationBadValue = true;
+            }
+        }
+
+        if (latBadValue) {
+            addRowError(Columns.LAT, "Invalid Lat value (expected range -90 to 90)", validationErrors, rowNum);
+        }
+
+        if (lonBadValue) {
+            addRowError(Columns.LONG, "Invalid Long value (expected range -180 to 180)", validationErrors, rowNum);
+        }
+
+        if (elevationBadValue) {
+            addRowError(Columns.LONG, "Invalid Elevation value (numerals expected)", validationErrors, rowNum);
+        }
+
     }
 
     private Map<String, BrAPIObservation> fetchExistingObservations(List<Trait> referencedTraits, Program program) throws ApiException {

src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java

@@ -32,15 +32,21 @@ public enum ExperimentFileColumns {
     EXP_TITLE(ExperimentObservation.Columns.EXP_TITLE, Column.ColumnDataType.STRING),
     EXP_DESCRIPTION(ExperimentObservation.Columns.EXP_DESCRIPTION, Column.ColumnDataType.STRING),
     EXP_UNIT(ExperimentObservation.Columns.EXP_UNIT, Column.ColumnDataType.STRING),
+    SUB_OBS_UNIT(ExperimentObservation.Columns.SUB_OBS_UNIT, Column.ColumnDataType.STRING),
     EXP_TYPE(ExperimentObservation.Columns.EXP_TYPE, Column.ColumnDataType.STRING),
     ENV(ExperimentObservation.Columns.ENV, Column.ColumnDataType.STRING),
     ENV_LOCATION(ExperimentObservation.Columns.ENV_LOCATION, Column.ColumnDataType.STRING),
     ENV_YEAR(ExperimentObservation.Columns.ENV_YEAR, Column.ColumnDataType.INTEGER),
     EXP_UNIT_ID(ExperimentObservation.Columns.EXP_UNIT_ID, Column.ColumnDataType.STRING),
+    SUB_UNIT_ID(ExperimentObservation.Columns.SUB_UNIT_ID, Column.ColumnDataType.STRING),
     REP_NUM(ExperimentObservation.Columns.REP_NUM, Column.ColumnDataType.INTEGER),
     BLOCK_NUM(ExperimentObservation.Columns.BLOCK_NUM, Column.ColumnDataType.INTEGER),
     ROW(ExperimentObservation.Columns.ROW, Column.ColumnDataType.STRING),
     COLUMN(ExperimentObservation.Columns.COLUMN, Column.ColumnDataType.STRING),
+    LAT(ExperimentObservation.Columns.LAT, Column.ColumnDataType.STRING),
+    LONG(ExperimentObservation.Columns.LONG, Column.ColumnDataType.STRING),
+    ELEVATION(ExperimentObservation.Columns.ELEVATION, Column.ColumnDataType.STRING),
+    RTK(ExperimentObservation.Columns.RTK, Column.ColumnDataType.STRING),
     TREATMENT_FACTORS(ExperimentObservation.Columns.TREATMENT_FACTORS, Column.ColumnDataType.STRING),
     OBS_UNIT_ID(ExperimentObservation.Columns.OBS_UNIT_ID, Column.ColumnDataType.STRING);
 

src/main/java/org/breedinginsight/utilities/FileUtil.java

@@ -173,7 +173,9 @@ public static Table parseTableFromCsv(InputStream inputStream) throws ParsingExc
             // Convert to Table.
             //Jackson used downstream messily converts LOCAL_DATE/LOCAL_DATETIME, so need to interpret date input as strings
             //Note that if another type is needed later this is what needs to be updated
-            ArrayList<ColumnType> acceptedTypes = new ArrayList<>(Arrays.asList(ColumnType.STRING, ColumnType.INTEGER, ColumnType.DOUBLE, ColumnType.FLOAT));
+            //Removed FLOAT and DOUBLE types because it was causing issues with experiment geocoordinates which are to be treated as strings
+            //until validations are done
+            ArrayList<ColumnType> acceptedTypes = new ArrayList<>(Arrays.asList(ColumnType.STRING, ColumnType.INTEGER));
             Table df = Table.read().usingOptions(
                     CsvReadOptions
                             .builderFromString(input)