[BI-1819] Experimental download file: phenotypes are out of order

src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java

@@ -17,6 +17,7 @@
 import org.breedinginsight.brapps.importer.model.exports.FileType;
 import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation;
 import org.breedinginsight.brapps.importer.services.ExternalReferenceSource;
+import org.breedinginsight.brapps.importer.services.FileMappingUtil;
 import org.breedinginsight.model.BrAPIConstants;
 import org.breedinginsight.model.Column;
 import org.breedinginsight.model.DownloadFile;
@@ -46,6 +47,7 @@ public class BrAPITrialService {
     private final BrAPISeasonDAO seasonDAO;
     private final BrAPIObservationUnitDAO ouDAO;
     private final BrAPIGermplasmDAO germplasmDAO;
+    private final FileMappingUtil fileMappingUtil;
 
     @Inject
     public BrAPITrialService(@Property(name = "brapi.server.reference-source") String referenceSource,
@@ -56,7 +58,8 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin
                              BrAPIStudyDAO studyDAO,
                              BrAPISeasonDAO seasonDAO,
                              BrAPIObservationUnitDAO ouDAO,
-                             BrAPIGermplasmDAO germplasmDAO) {
+                             BrAPIGermplasmDAO germplasmDAO,
+                             FileMappingUtil fileMappingUtil) {
 
         this.referenceSource = referenceSource;
         this.trialDAO = trialDAO;
@@ -67,6 +70,7 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin
         this.seasonDAO = seasonDAO;
         this.ouDAO = ouDAO;
         this.germplasmDAO = germplasmDAO;
+        this.fileMappingUtil = fileMappingUtil;
     }
 
     public List<BrAPITrial> getExperiments(UUID programId) throws ApiException, DoesNotExistException {
@@ -275,7 +279,10 @@ public List<BrAPIObservationVariable> getDatasetObsVars(String datasetId, Progra
         String listDbId = lists.get(0).getListDbId();
         BrAPIListsSingleResponse list = listDAO.getListById(listDbId, program.getId());
         List<String> obsVarNames = list.getResult().getData();
-        return obsVarDAO.getVariableByName(obsVarNames, program.getId());
+        List<BrAPIObservationVariable> obsVars = obsVarDAO.getVariableByName(obsVarNames, program.getId());
+
+        // sort the obsVars to match the order stored in the dataset list
+        return fileMappingUtil.sortByField(obsVarNames, obsVars, BrAPIObservationVariable::getObservationVariableName);
     }
 
     public BrAPITrial getExperiment(Program program, UUID experimentId) throws ApiException {

src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java

@@ -17,30 +17,27 @@
 
 package org.breedinginsight.brapps.importer.services;
 
-import io.micronaut.http.server.exceptions.InternalServerException;
+import io.reactivex.functions.Function;
 import org.apache.commons.lang3.tuple.MutablePair;
 import org.apache.commons.lang3.tuple.Pair;
-import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO;
-import org.breedinginsight.brapps.importer.daos.*;
 import org.breedinginsight.brapps.importer.model.config.MappedImportRelation;
-import org.breedinginsight.brapps.importer.model.mapping.ImportMapping;
-import org.breedinginsight.brapps.importer.model.mapping.MappingField;
-import org.jooq.DSLContext;
 import tech.tablesaw.api.Row;
 import tech.tablesaw.api.Table;
-import tech.tablesaw.columns.Column;
 
 import javax.inject.Inject;
 import javax.inject.Singleton;
-import java.util.*;
-import java.util.stream.Collectors;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 @Singleton
 public class FileMappingUtil {
 
     public static final String EXPERIMENT_TEMPLATE_NAME = "ExperimentsTemplateMap";
     private FileImportService fileImportService;
 
+
     @Inject
     public FileMappingUtil(FileImportService fileImportService) {
         this.fileImportService = fileImportService;
@@ -77,27 +74,23 @@ public List<Pair<Integer, String>> findFileRelationships(Table data, List<Mapped
 
         return targetIndexList;
     }
-    public List<Column<?>> getDynamicColumns(Table data, String templateName) {
-        List<ImportMapping> result = fileImportService.getSystemMappingByName(templateName);
 
-        if (result.isEmpty()) {
-            throw new InternalServerException("System mapping does not exist");
+    public <T> List<T> sortByField(List<String> sortedFields, List<T> unsortedItems, Function<T, String> fieldGetter) {
+        HashMap<String, Integer> sortOrder = new HashMap<>();
+        for (int i = 0; i < sortedFields.size(); i++) {
+            sortOrder.put(sortedFields.get(i), i);
         }
 
-        ImportMapping mapping = result.get(0);
-        List<MappingField> config = mapping.getMappingConfig();
-        List<String> columnNames = new ArrayList<>();
-
-        for (MappingField field : config) {
-            if (field.getValue() != null) {
-                columnNames.add(field.getValue().getFileFieldName());
+        unsortedItems.sort((i1, i2) -> {
+            try {
+                String field1 = fieldGetter.apply(i1);
+                String field2 = fieldGetter.apply(i2);
+                return Integer.compare(sortOrder.get(field1), sortOrder.get(field2));
+            } catch (Exception e) {
+                throw new RuntimeException(e);
             }
-        }
-
-        List<String> differences = data.columnNames().stream()
-                .filter(col -> !columnNames.contains(col))
-                .collect(Collectors.toList());
+        });
 
-        return data.columns(differences.toArray(String[]::new));
+        return unsortedItems;
     }
 }

src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java

@@ -195,6 +195,7 @@ public void getExistingBrapiData(List<BrAPIImport> importRows, Program program)
      */
     @Override
     public Map<String, ImportPreviewStatistics> process(
+            ImportUpload upload,
             List<BrAPIImport> importRows,
             Map<Integer, PendingImport> mappedBrAPIImport,
             Table data,
@@ -206,7 +207,7 @@ public Map<String, ImportPreviewStatistics> process(
         ValidationErrors validationErrors = new ValidationErrors();
 
         // Get dynamic phenotype columns for processing
-        List<Column<?>> dynamicCols = fileMappingUtil.getDynamicColumns(data, EXPERIMENT_TEMPLATE_NAME);
+        List<Column<?>> dynamicCols = data.columns(upload.getDynamicColumnNames());
         List<Column<?>> phenotypeCols = new ArrayList<>();
         List<Column<?>> timestampCols = new ArrayList<>();
         for (Column<?> dynamicCol : dynamicCols) {
@@ -436,7 +437,9 @@ private List<Trait> verifyTraits(UUID programId, List<Column<?>> phenotypeCols,
                                           "Timestamp column(s) lack corresponding phenotype column(s): " + String.join(COMMA_DELIMITER, unmatchedTimestamps));
         }
 
-        return filteredTraits;
+        // sort the verified traits to match the order of the trait columns
+        List<String> phenotypeColNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toList());
+        return fileMappingUtil.sortByField(phenotypeColNames, filteredTraits, TraitEntity::getObservationVariableName);
     }
 
     private List<Trait> fetchFileTraits(UUID programId, Collection<String> varNames) {

src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java

@@ -16,9 +16,6 @@
  */
 package org.breedinginsight.brapps.importer.services.processors;
 
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
 import io.micronaut.context.annotation.Property;
 import io.micronaut.context.annotation.Prototype;
 import io.micronaut.http.HttpStatus;
@@ -30,7 +27,6 @@
 import org.brapi.v2.model.core.BrAPIListSummary;
 import org.brapi.v2.model.core.request.BrAPIListNewRequest;
 import org.brapi.v2.model.germ.BrAPIGermplasm;
-import org.brapi.v2.model.germ.BrAPIGermplasmSynonyms;
 import org.breedinginsight.api.model.v1.response.ValidationError;
 import org.breedinginsight.api.model.v1.response.ValidationErrors;
 import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields;
@@ -221,7 +217,7 @@ public void getExistingBrapiData(List<BrAPIImport> importRows, Program program)
     }
 
     @Override
-    public Map<String, ImportPreviewStatistics> process(List<BrAPIImport> importRows,
+    public Map<String, ImportPreviewStatistics> process(ImportUpload upload, List<BrAPIImport> importRows,
                                                         Map<Integer, PendingImport> mappedBrAPIImport, Table data,
                                                         Program program, User user, boolean commit) throws ValidatorException {
 

src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java

@@ -78,7 +78,7 @@ public void getExistingBrapiData(List<BrAPIImport> importRows, Program program)
     }
 
     @Override
-    public Map<String, ImportPreviewStatistics> process(List<BrAPIImport> importRows,
+    public Map<String, ImportPreviewStatistics> process(ImportUpload upload, List<BrAPIImport> importRows,
                                                         Map<Integer, PendingImport> mappedBrAPIImport, Table data,
                                                         Program program, User user, boolean commit) throws ValidatorException {
 

src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java

@@ -142,7 +142,7 @@ private void getDependentDbIds(List<BrAPIImport> importRows, Program program) {
     }
 
     @Override
-    public Map<String, ImportPreviewStatistics> process(List<BrAPIImport> importRows,
+    public Map<String, ImportPreviewStatistics> process(ImportUpload upload, List<BrAPIImport> importRows,
                                                         Map<Integer,PendingImport> mappedBrAPIImport, Table data,
                                                         Program program, User user, boolean commit) throws ValidatorException {
 

src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java

@@ -88,7 +88,7 @@ private void getExistingStudyObjects() {
     }
 
     @Override
-    public Map<String, ImportPreviewStatistics> process(List<BrAPIImport> importRows,
+    public Map<String, ImportPreviewStatistics> process(ImportUpload upload, List<BrAPIImport> importRows,
                                                         Map<Integer,PendingImport> mappedBrAPIImport, Table data,
                                                         Program program, User user, boolean commit) {
 

src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java

@@ -43,14 +43,16 @@ public interface Processor {
     /**
      * Update mappedBrAPIImport mapping with PendingImport data for brapi object based on new and existing objects.
      * Return stats on number of new & existing objects
+     *
+     * @param upload
      * @param importRows
      * @param mappedBrAPIImport
      * @param data
      * @param program
      * @return
      * @throws ValidatorException
      */
-    Map<String, ImportPreviewStatistics> process(List<BrAPIImport> importRows,
+    Map<String, ImportPreviewStatistics> process(ImportUpload upload, List<BrAPIImport> importRows,
                                                  Map<Integer, PendingImport> mappedBrAPIImport, Table data,
                                                  Program program, User user, boolean commit)
             throws ValidatorException, MissingRequiredInfoException, ApiException;

src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java

@@ -62,7 +62,7 @@ public ImportPreviewResponse process(List<BrAPIImport> importRows, List<Processo
             log.debug("Checking existing " + processor.getName().toLowerCase() + " objects in brapi service and mapping data");
             statusService.updateMessage(upload, "Checking existing " + processor.getName().toLowerCase() + " objects in brapi service and mapping data");
             processor.getExistingBrapiData(importRows, program);
-            Map<String, ImportPreviewStatistics> stats = processor.process(importRows, mappedBrAPIImport, data, program, user, commit);
+            Map<String, ImportPreviewStatistics> stats = processor.process(upload, importRows, mappedBrAPIImport, data, program, user, commit);
             statistics.putAll(stats);
         }
 

src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java

@@ -21,7 +21,6 @@
 import io.micronaut.http.exceptions.HttpStatusException;
 import io.micronaut.http.server.exceptions.InternalServerException;
 import org.brapi.client.v2.model.exceptions.ApiException;
-import org.brapi.v2.model.core.BrAPILocation;
 import org.brapi.v2.model.core.BrAPIStudy;
 import org.brapi.v2.model.core.BrAPITrial;
 import org.breedinginsight.brapps.importer.daos.BrAPIStudyDAO;
@@ -80,7 +79,7 @@ public void getExistingBrapiData(List<BrAPIImport> importRows, Program program)
     }
 
     @Override
-    public Map<String, ImportPreviewStatistics> process(List<BrAPIImport> importRows,
+    public Map<String, ImportPreviewStatistics> process(ImportUpload upload, List<BrAPIImport> importRows,
                                                         Map<Integer, PendingImport> mappedBrAPIImport, Table data,
                                                         Program program, User user, boolean commit) {
 

src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java

@@ -74,7 +74,7 @@ public void getExistingBrapiData(List<BrAPIImport> importRows, Program program)
     }
 
     @Override
-    public Map<String, ImportPreviewStatistics> process(List<BrAPIImport> importRows,
+    public Map<String, ImportPreviewStatistics> process(ImportUpload upload, List<BrAPIImport> importRows,
                                                         Map<Integer, PendingImport> mappedBrAPIImport, Table data,
                                                         Program program, User user, boolean commit) throws ValidatorException {