BI-2053 - Fixed Scale Units Bug

src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java

@@ -380,6 +380,7 @@ public List<Trait> createTraitsBrAPI(List<Trait> traits, User actingUser, Progra
                                                                                .min(trait.getScale().getValidValueMin());
             BrAPIScale brApiScale = new BrAPIScale()
                     .scaleName(String.format("%s [%s]", trait.getScale().getScaleName(), program.getKey()))
+                    .units(trait.getScale().getUnits())
                     .externalReferences(List.of(scaleReference))
                     .dataType(brApiTraitDataType)
                     .decimalPlaces(trait.getScale().getDecimalPlaces())

src/main/java/org/breedinginsight/model/Scale.java

@@ -58,6 +58,7 @@ public class Scale extends ScaleEntity {
     public Scale(ScaleEntity scaleEntity){
         this.setId(scaleEntity.getId());
         this.setScaleName(scaleEntity.getScaleName());
+        this.setUnits(scaleEntity.getUnits());
         this.setProgramOntologyId(scaleEntity.getProgramOntologyId());
         this.setDataType(scaleEntity.getDataType());
         this.setCreatedAt(scaleEntity.getCreatedAt());
@@ -70,6 +71,7 @@ public static Scale parseSqlRecord(Record record) {
         return Scale.builder()
             .id(record.getValue(SCALE.ID))
             .scaleName(record.getValue(SCALE.SCALE_NAME))
+            .units(record.getValue(SCALE.UNITS))
             .programOntologyId(record.getValue(SCALE.PROGRAM_ONTOLOGY_ID))
             .dataType(record.getValue(SCALE.DATA_TYPE))
             .createdAt(record.getValue(SCALE.CREATED_AT))

src/main/java/org/breedinginsight/services/OntologyService.java

@@ -505,7 +505,7 @@ public List<Map<String, Object>> processData(List<Trait> traits) {
             if(scale!=null) {
 
                 row.put(TraitFileColumns.SCALE_CLASS.toString(), DataTypeTranslator.getDisplayNameFromType(scale.getDataType()));
-                row.put(TraitFileColumns.SCALE_NAME.toString(), scale.getScaleName());
+                row.put(TraitFileColumns.UNITS.toString(), scale.getUnits());
                 row.put(TraitFileColumns.SCALE_DECIMAL_PLACES.toString(), scale.getDecimalPlaces());
                 row.put(TraitFileColumns.SCALE_LOWER_LIMIT.toString(), scale.getValidValueMin());
                 row.put(TraitFileColumns.SCALE_UPPER_LIMIT.toString(), scale.getValidValueMax());

src/main/java/org/breedinginsight/services/TraitService.java

@@ -22,19 +22,10 @@
 import io.micronaut.http.server.exceptions.HttpServerException;
 import io.micronaut.http.server.exceptions.InternalServerException;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
 import org.apache.commons.text.WordUtils;
-import org.brapi.client.v2.model.exceptions.ApiException;
-import org.brapi.v2.model.BrAPIOntologyReference;
-import org.brapi.v2.model.core.BrAPIStudy;
-import org.brapi.v2.model.core.BrAPITrial;
-import org.brapi.v2.model.pheno.*;
+import org.brapi.v2.model.pheno.BrAPIObservation;
 import org.breedinginsight.api.auth.AuthenticatedUser;
 import org.breedinginsight.api.model.v1.response.ValidationErrors;
-import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields;
-import org.breedinginsight.brapi.v2.services.BrAPITrialService;
-import org.breedinginsight.brapps.importer.services.ExternalReferenceSource;
 import org.breedinginsight.dao.db.enums.DataType;
 import org.breedinginsight.dao.db.tables.pojos.MethodEntity;
 import org.breedinginsight.dao.db.tables.pojos.ProgramSharedOntologyEntity;
@@ -46,8 +37,6 @@
 import org.breedinginsight.services.exceptions.ValidatorException;
 import org.breedinginsight.services.validators.TraitValidatorError;
 import org.breedinginsight.services.validators.TraitValidatorService;
-import org.breedinginsight.utilities.Utilities;
-import org.jetbrains.annotations.NotNull;
 import org.jooq.DSLContext;
 
 import javax.inject.Inject;
@@ -203,6 +192,7 @@ public List<Trait> createTraits(UUID programId, List<Trait> traits, Authenticate
                     // Create scale
                     ScaleEntity jooqScale = ScaleEntity.builder()
                             .scaleName(trait.getScale().getScaleName())
+                            .units(trait.getScale().getUnits())
                             .dataType(trait.getScale().getDataType())
                             .programOntologyId(programOntology.getId())
                             .createdBy(actingUser.getId())
@@ -419,6 +409,7 @@ public List<Trait> updateTraits(UUID programId, List<Trait> traits, Authenticate
                     // Jump to scale
                     ScaleEntity existingScaleEntity = scaleDAO.fetchOneById(existingTraitEntity.getScaleId());
                     existingScaleEntity.setScaleName(updatedTrait.getScale().getScaleName());
+                    existingScaleEntity.setUnits(updatedTrait.getScale().getUnits());
                     existingScaleEntity.setDataType(updatedTrait.getScale().getDataType());
                     existingScaleEntity.setUpdatedBy(user.getId());
                     scaleDAO.update(existingScaleEntity);

src/main/java/org/breedinginsight/services/parsers/ParsingExceptionType.java

@@ -36,7 +36,8 @@ public enum ParsingExceptionType {
     INVALID_SCALE_LOWER_LIMIT("Invalid scale lower limit value. Value must be numeric and be a whole number."),
     INVALID_SCALE_UPPER_LIMIT("Invalid scale upper limit value. Value must be numeric and be a whole number."),
     INVALID_SCALE_CATEGORIES("Invalid scale categories format"),
-    INVALID_TERM_TYPE("Invalid term type");
+    INVALID_TERM_TYPE("Invalid term type"),
+    SCALE_UNIT_NOT_ALLOWED("Units identified for non-numeric scale class");
 
     private String value;
 

src/main/java/org/breedinginsight/services/parsers/trait/TraitFileColumns.java

@@ -38,7 +38,7 @@ public enum TraitFileColumns {
     METHOD_CLASS("Method class", Column.ColumnDataType.STRING),
     METHOD_FORMULA("Method formula", Column.ColumnDataType.STRING),
     SCALE_CLASS("Scale class", Column.ColumnDataType.STRING),
-    SCALE_NAME("Units", Column.ColumnDataType.STRING),
+    UNITS("Units", Column.ColumnDataType.STRING),
     SCALE_DECIMAL_PLACES("Scale decimal places", Column.ColumnDataType.INTEGER),
     SCALE_LOWER_LIMIT("Scale lower limit", Column.ColumnDataType.INTEGER),
     SCALE_UPPER_LIMIT("Scale upper limit", Column.ColumnDataType.INTEGER),

src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java

@@ -242,8 +242,18 @@ private List<Trait> excelRecordsToTraits(List<ExcelRecord> records) throws Valid
                 }
             }
 
+            String units = parseExcelValueAsString(record, TraitFileColumns.UNITS);
+            // Note: if method class is "Computation", scale class will be overwritten to "Numerical", so treat accordingly.
+            if (dataType != DataType.NUMERICAL && (method.getMethodClass() == null || !method.getMethodClass().equalsIgnoreCase(Method.COMPUTATION_TYPE))) {
+                if (units != null && !units.isEmpty()) {
+                    ValidationError error = new ValidationError(TraitFileColumns.UNITS.toString(),
+                            ParsingExceptionType.SCALE_UNIT_NOT_ALLOWED.toString(), HttpStatus.UNPROCESSABLE_ENTITY);
+                    validationErrors.addError(traitValidatorError.getRowNumber(i), error);
+                }
+            }
             Scale scale = Scale.builder()
-                    .scaleName(parseExcelValueAsString(record, TraitFileColumns.SCALE_NAME))
+                    .scaleName(units == null ? parseExcelValueAsString(record, TraitFileColumns.SCALE_CLASS) : units)
+                    .units(units)
                     .dataType(dataType)
                     .decimalPlaces(decimalPlaces)
                     .validValueMin(validValueMin)

src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java

@@ -56,7 +56,7 @@ public ValidationError getMissingScaleMsg() {
     }
 
     @Override
-    public ValidationError getMissingScaleNameMsg() {
+    public ValidationError getMissingScaleUnitMsg() {
         return new ValidationError("Unit", "Missing unit", HttpStatus.UNPROCESSABLE_ENTITY);
     }
 

src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java

@@ -56,8 +56,8 @@ public ValidationError getMissingScaleMsg() {
     }
 
     @Override
-    public ValidationError getMissingScaleNameMsg() {
-        return new ValidationError("scale.scaleName", "Missing scale name", HttpStatus.BAD_REQUEST);
+    public ValidationError getMissingScaleUnitMsg() {
+        return new ValidationError("scale.units", "Missing units", HttpStatus.BAD_REQUEST);
     }
 
     @Override

src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java

@@ -28,7 +28,7 @@ public interface TraitValidatorErrorInterface {
     ValidationError getMissingMethodMsg();
     ValidationError getMissingMethodClassMsg();
     ValidationError getMissingScaleMsg();
-    ValidationError getMissingScaleNameMsg();
+    ValidationError getMissingScaleUnitMsg();
     ValidationError getMissingScaleDataTypeMsg();
     ValidationError getMissingObsVarNameMsg();
     ValidationError getMissingTraitEntityMsg();

src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java

@@ -18,7 +18,6 @@
 
 import org.breedinginsight.api.model.v1.response.ValidationError;
 import org.breedinginsight.api.model.v1.response.ValidationErrors;
-import org.breedinginsight.brapi.v1.model.TraitDataType;
 import org.breedinginsight.dao.db.enums.DataType;
 import org.breedinginsight.daos.TraitDAO;
 import org.breedinginsight.model.Method;
@@ -65,8 +64,8 @@ public ValidationErrors checkRequiredTraitFields(List<Trait> traits, TraitValida
                 errors.addError(traitValidatorErrors.getRowNumber(i), error);
             } else {
                 if (scale.getDataType() != null & scale.getDataType() == DataType.NUMERICAL &&
-                        (isBlank(scale.getScaleName()) || scale.getScaleName() == null )) {
-                    ValidationError error = traitValidatorErrors.getMissingScaleNameMsg();
+                        (isBlank(scale.getUnits()))) {
+                    ValidationError error = traitValidatorErrors.getMissingScaleUnitMsg();
                     errors.addError(traitValidatorErrors.getRowNumber(i), error);
                 }
                 if (scale.getDataType() == null || scale.getDataType() == null) {

src/main/resources/db/migration/V1.18.0__alter_scale_table.sql

@@ -0,0 +1,23 @@
+/*
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Add units to scale table, units is an optional BrAPI field.
+ALTER TABLE scale ADD COLUMN units text;
+-- Update existing scale records.
+UPDATE scale
+SET units = scale_name
+WHERE data_type = 'NUMERICAL';

src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java

@@ -1367,6 +1367,8 @@ public void postTraitComputation() {
 
         // Set scale class to computation
         trait1.getMethod().setMethodClass("Computation");
+        // Since method class of "Computation" will change scale class to "Numerical", units is reauired.
+        trait1.getScale().setUnits("test unit");
 
         List<Trait> traits = List.of(trait1);
 

src/test/java/org/breedinginsight/api/v1/controller/TraitUploadControllerIntegrationTest.java

@@ -581,7 +581,7 @@ private void checkValidTraitUpload(JsonObject traitUpload) {
         assertEquals("a^2 + b^2 = c^2", method.get("formula").getAsString(), "wrong method formula");
 
         JsonObject scale = trait.get("scale").getAsJsonObject();
-        assertEquals("1-4 Parlier field response score", scale.get("scaleName").getAsString(), "wrong scale name");
+        assertEquals("Ordinal", scale.get("scaleName").getAsString(), "wrong scale name");
         assertEquals(DataType.ORDINAL.getLiteral(), scale.get("dataType").getAsString(), "wrong scale dataType");
         assertEquals(2, scale.get("decimalPlaces").getAsInt(), "wrong scale decimal places");
         assertEquals(2, scale.get("validValueMin").getAsInt(), "wrong scale min value");
@@ -811,6 +811,27 @@ void putNominalCategoryLabelPresentError() {
         assertEquals("Scale label cannot be populated for Nominal scale type", labelError.get("errorMessage").getAsString());
     }
 
+    @Test
+    void putNonNumericalClassWithUnitError() {
+
+        File file = new File("src/test/resources/files/ontology/non_numerical_with_unit.xlsx");
+
+        HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> {
+            HttpResponse<String> response = uploadFile(validProgram.getId().toString(), file, "test-registered-user");
+        });
+        assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, e.getStatus());
+
+        JsonArray rowErrors = JsonParser.parseString((String) e.getResponse().getBody().get()).getAsJsonObject().getAsJsonArray("rowErrors");
+        assertTrue(rowErrors.size() == 1, "Wrong number of row errors returned");
+
+        JsonObject rowError1 = rowErrors.get(0).getAsJsonObject();
+        JsonArray errors = rowError1.getAsJsonArray("errors");
+        assertTrue(errors.size() == 1, "Not enough errors were returned");
+        JsonObject error = errors.get(0).getAsJsonObject();
+        assertEquals(422, error.get("httpStatusCode").getAsInt(), "Incorrect http status code");
+        assertEquals("Units identified for non-numeric scale class", error.get("errorMessage").getAsString(), "Incorrect http status code");
+    }
+
     @Test
     void putOrdinalMissingCategories() {
 

src/test/java/org/breedinginsight/brapi/v1/controller/BrapiV1ObservationVariablesControllerIntegrationTest.java

@@ -163,6 +163,7 @@ Trait buildTestTrait(String name, String traitClass, String description,
 
         Scale scale = Scale.builder()
                 .scaleName(name + " scale")
+                .units("test unit")
                 .dataType(DataType.NUMERICAL)
                 .build();
 

src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java

@@ -290,6 +290,7 @@ private List<Trait> createTraits(int numToCreate) {
                     .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build())
                     .scale(Scale.builder()
                             .scaleName("test scale")
+                            .units("test unit")
                             .dataType(DataType.NUMERICAL)
                             .validValueMin(0)
                             .validValueMax(100)

src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java

@@ -422,6 +422,7 @@ private List<Trait> createTraits(int numToCreate) {
                     .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build())
                     .scale(Scale.builder()
                             .scaleName("test scale")
+                            .units("test unit")
                             .dataType(DataType.NUMERICAL)
                             .validValueMin(0)
                             .validValueMax(100)

src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java

@@ -378,6 +378,7 @@ private List<Trait> createTraits(int numToCreate) {
                     .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build())
                     .scale(Scale.builder()
                             .scaleName("test scale")
+                            .units("test unit")
                             .dataType(DataType.NUMERICAL)
                             .validValueMin(0)
                             .validValueMax(100)

src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java

@@ -376,6 +376,7 @@ private List<Trait> createTraits(int numToCreate) {
                     .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build())
                     .scale(Scale.builder()
                             .scaleName("test scale")
+                            .units("test unit")
                             .dataType(DataType.NUMERICAL)
                             .validValueMin(0)
                             .validValueMax(100)

src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java

@@ -386,6 +386,7 @@ private List<Trait> createTraits(int numToCreate) {
                     .scale(Scale.builder()
                             .scaleName("test scale")
                             .dataType(DataType.NUMERICAL)
+                            .units("test unit")
                             .validValueMin(0)
                             .validValueMax(100)
                             .build())

src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java

@@ -314,6 +314,7 @@ private List<Trait> createTraits(int numToCreate) {
                     .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build())
                     .scale(Scale.builder()
                             .scaleName("test scale")
+                            .units("test unit")
                             .dataType(DataType.NUMERICAL)
                             .validValueMin(0)
                             .validValueMax(100)

src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java

@@ -182,6 +182,7 @@ public List<Trait> createTraits(int numToCreate) {
                     .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build())
                     .scale(Scale.builder()
                             .scaleName("test scale")
+                            .units("test unit")
                             .dataType(DataType.NUMERICAL)
                             .validValueMin(0)
                             .validValueMax(100)

src/test/java/org/breedinginsight/services/parsers/TraitFileParserUnitTest.java

@@ -295,7 +295,7 @@ private void assertTestTraitEquals(Trait trait) {
         assertEquals("Estimation", method.getMethodClass(), "wrong method class");
         assertEquals("a^2 + b^2 = c^2", method.getFormula(), "wrong method formula");
         Scale scale = trait.getScale();
-        assertEquals("1-4 Parlier field response score", scale.getScaleName(), "wrong scale name");
+        assertEquals("Ordinal", scale.getScaleName(), "wrong scale name");
         assertEquals(DataType.ORDINAL, scale.getDataType(), "wrong scale dataType");
         assertEquals(2, scale.getDecimalPlaces(), "wrong scale decimal places");
         assertEquals(2, scale.getValidValueMin(), "wrong scale min value");

src/test/resources/files/data_mismatched_cases.csv

@@ -1,2 +1,2 @@
 name,synoNyms,deScription,Trait entity,Trait attribute,StaTus,Trait lists,Method name,Method description,Method class,Method formula,Units,Scale class,Scale decimal places,Scale lower limit,ScALe Upper limit,Scale CAtegories,Tags,Full Name,Term Type
-PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,,Germplasm PASSPORT
+PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,,Germplasm PASSPORT