[BI-1993] - Accept NA observations

pom.xml

@@ -97,7 +97,7 @@
         <apache-poi-ooxml.version>4.1.2</apache-poi-ooxml.version>
         <javaxmail.version>1.6.2</javaxmail.version>
         <st4.version>4.3.1</st4.version>
-        <tablesaw.version>0.42.0</tablesaw.version>
+        <tablesaw.version>1.0.0-SNAPSHOT</tablesaw.version>
         <alphanumeric-comparator.version>1.4.1</alphanumeric-comparator.version>
         <cloning.version>1.10.3</cloning.version>
     </properties>
@@ -160,6 +160,13 @@
                 <enabled>false</enabled>
             </releases>
         </repository>
+        <repository>
+            <id>github-tablesaw</id>
+            <name>Tablesaw github repository</name>
+            <url>https://maven.pkg.github.com/Breeding-Insight/tablesaw</url>
+            <releases><enabled>true</enabled></releases>
+            <snapshots><enabled>true</enabled></snapshots>
+        </repository>
     </repositories>
 
     <dependencyManagement>

settings.xml

@@ -56,6 +56,13 @@
                     <releases><enabled>true</enabled></releases>
                     <snapshots><enabled>true</enabled></snapshots>
                 </repository>
+                <repository>
+                    <id>github-tablesaw</id>
+                    <name>Tablesaw github repository</name>
+                    <url>https://maven.pkg.github.com/Breeding-Insight/tablesaw</url>
+                    <releases><enabled>true</enabled></releases>
+                    <snapshots><enabled>true</enabled></snapshots>
+                </repository>
             </repositories>
         </profile>
     </profiles>
@@ -71,5 +78,10 @@
             <username>${GITHUB_ACTOR}</username>
             <password>${GITHUB_TOKEN}</password>
         </server>
+        <server>
+            <id>github-tablesaw</id>
+            <username>${GITHUB_ACTOR}</username>
+            <password>${GITHUB_TOKEN}</password>
+        </server>
     </servers>
 </settings>

src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java

@@ -390,8 +390,8 @@ private void addObsVarDataToRow(
             Trait var,
             Program program) {
         String varName = Utilities.removeProgramKey(obs.getObservationVariableName(), program.getKey());
-        if (var.getScale().getDataType().equals(DataType.NUMERICAL) ||
-                var.getScale().getDataType().equals(DataType.DURATION)) {
+        if (!(obs.getValue().equalsIgnoreCase("NA")) && (var.getScale().getDataType().equals(DataType.NUMERICAL) ||
+                var.getScale().getDataType().equals(DataType.DURATION))) {
             row.put(varName, Double.parseDouble(obs.getValue()));
         } else {
             row.put(varName, obs.getValue());

src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java

@@ -1773,13 +1773,22 @@ private void validateTimeStampValue(String value,
 
     }
 
+    private boolean isNAObservation(String value){
+        return value.equalsIgnoreCase("NA");
+    }
+
     private void validateObservationValue(Trait variable, String value,
                                           String columnHeader, ValidationErrors validationErrors, int row) {
         if (StringUtils.isBlank(value)) {
             log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s\n\trow: %d", variable.getObservationVariableName(), row));
             return;
         }
 
+        if (isNAObservation(value)) {
+            log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s\n\trow: %d", variable.getObservationVariableName(), row));
+            return;
+        }
+
         switch (variable.getScale().getDataType()) {
             case NUMERICAL:
                 Optional<BigDecimal> number = validNumericValue(value);

src/main/java/org/breedinginsight/services/writers/ExcelWriter.java

@@ -54,7 +54,8 @@ public static Workbook writeToWorkbook(String sheetName, List<Column> columns, L
                 } else {
                     //Data values
                     if (data.get(i-1).get(column.getValue()) != null) {
-                        if (column.getDataType() == Column.ColumnDataType.STRING) {
+                        //If String or NA value, accept as is
+                        if (column.getDataType() == Column.ColumnDataType.STRING || data.get(i-1).get(column.getValue()).toString().equalsIgnoreCase("NA")) {
                             row.createCell(cellCount).setCellValue((String) data.get(i - 1).get(column.getValue()));
                         } else if (column.getDataType() == Column.ColumnDataType.INTEGER) {
                             row.createCell(cellCount).setCellValue((Integer) data.get(i - 1).get(column.getValue()));

src/main/java/org/breedinginsight/utilities/FileUtil.java

@@ -192,17 +192,12 @@ public static Table parseTableFromCsv(InputStream inputStream) throws ParsingExc
     }
 
     public static Table parseTableFromJson(String jsonString) throws ParsingException {
-        try {
             return Table.read()
                     .usingOptions(
                             JsonReadOptions
                                     .builderFromString(jsonString)
                                     .columnTypesToDetect(List.of(ColumnType.STRING))
                     );
-        } catch (IOException e) {
-            throw new ParsingException(ParsingExceptionType.ERROR_READING_FILE);
-        }
-
     }
 
     public static Table removeNullRows(Table table) {

src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java

@@ -17,7 +17,6 @@
 
 package org.breedinginsight.api.v1.controller;
 
-import com.eclipsesource.json.Json;
 import com.google.gson.*;
 import com.google.gson.reflect.TypeToken;
 import io.kowalski.fannypack.FannyPack;
@@ -41,10 +40,8 @@
 import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO;
 import org.breedinginsight.brapps.importer.ImportTestUtils;
 import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation;
-import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport;
 import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport.Columns;
 import org.breedinginsight.brapps.importer.services.ExternalReferenceSource;
-import org.breedinginsight.dao.db.tables.pojos.ProgramBreedingMethodEntity;
 import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity;
 import org.breedinginsight.daos.SpeciesDAO;
 import org.breedinginsight.daos.UserDAO;
@@ -58,12 +55,10 @@
 import org.jetbrains.annotations.NotNull;
 import org.jooq.DSLContext;
 import org.junit.jupiter.api.*;
-import org.testcontainers.containers.localstack.LocalStackContainer;
 import tech.tablesaw.api.Table;
 
 import javax.inject.Inject;
 import java.io.*;
-import java.time.OffsetDateTime;
 import java.util.*;
 
 import static io.micronaut.http.HttpRequest.*;
@@ -76,15 +71,8 @@
 public class SampleSubmissionControllerIntegrationTest extends BrAPITest {
 
     private Program program;
-
     private ImportTestUtils importTestUtils;
 
-    private String experimentId;
-    private List<String> envIds = new ArrayList<>();
-    private final List<Map<String, Object>> rows = new ArrayList<>();
-    private final List<Column> columns = ExperimentFileColumns.getOrderedColumns();
-    private List<Trait> traits;
-
     @Property(name = "brapi.server.reference-source")
     private String BRAPI_REFERENCE_SOURCE;
     @Inject