#6920 Informative error message for custom metadata block definition.

src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java

@@ -18,6 +18,7 @@
 import java.io.FileReader;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import javax.ejb.EJB;
 import javax.ejb.EJBException;
@@ -32,10 +33,11 @@
 import javax.ws.rs.PathParam;
 import javax.ws.rs.core.Response;
 
-import org.apache.commons.io.IOUtils;
+import edu.harvard.iq.dataverse.util.BundleUtil;
 import org.apache.commons.lang.StringUtils;
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.asJsonArray;
 import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder;
+
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import javax.persistence.NoResultException;
@@ -232,8 +234,7 @@ public Response loadNAControlledVocabularyValue() {
         }
     }
 
-    private enum HeaderType {
-
+    public enum HeaderType {
         METADATABLOCK, DATASETFIELD, CONTROLLEDVOCABULARY
     }
 
@@ -298,12 +299,20 @@ public Response loadDatasetFields(File file) {
             alr.setActionResult(ActionLogRecord.Result.BadRequest);
             alr.setInfo( alr.getInfo() + "// file not found");
             return error(Status.EXPECTATION_FAILED, "File not found");
-
+
+        } catch (ArrayIndexOutOfBoundsException e) {
+            String message = getArrayIndexOutOfBoundMessage(header, lineNumber, e);
+            logger.log(Level.WARNING, message, e);
+            alr.setActionResult(ActionLogRecord.Result.InternalError);
+            alr.setInfo(alr.getInfo() + "// " + message);
+            return error(Status.INTERNAL_SERVER_ERROR, message);
+
         } catch (Exception e) {
-            logger.log(Level.WARNING, "Error parsing dataset fields:" + e.getMessage(), e);
+            String message = getGeneralErrorMessage(header, lineNumber, e.getMessage());
+            logger.log(Level.WARNING, message, e);
             alr.setActionResult(ActionLogRecord.Result.InternalError);
-            alr.setInfo( alr.getInfo() + "// " + e.getMessage());
-            return error(Status.INTERNAL_SERVER_ERROR, e.getMessage());
+            alr.setInfo( alr.getInfo() + "// " + message);
+            return error(Status.INTERNAL_SERVER_ERROR, message);
 
         } finally {
             if (br != null) {
@@ -319,6 +328,68 @@ public Response loadDatasetFields(File file) {
         return ok( Json.createObjectBuilder().add("added", responseArr) );
     }
 
+    /**
+     * Provide a general error message including the part and line number
+     * @param header
+     * @param lineNumber
+     * @param message
+     * @return
+     */
+    public String getGeneralErrorMessage(HeaderType header, int lineNumber, String message) {
+        List<String> arguments = new ArrayList<>();
+        arguments.add(header.name());
+        arguments.add(String.valueOf(lineNumber));
+        arguments.add(message);
+        return BundleUtil.getStringFromBundle("api.admin.datasetfield.load.GeneralErrorMessage", arguments);
+    }
+
+    /**
+     * Turn ArrayIndexOutOfBoundsException into an informative error message
+     * @param lineNumber
+     * @param header
+     * @param e
+     * @return
+     */
+    public String getArrayIndexOutOfBoundMessage(HeaderType header,
+                                                 int lineNumber,
+                                                 ArrayIndexOutOfBoundsException e) {
+
+        List<String> columns = getColumnsByHeader(header);
+        int wrongIndex = Integer.parseInt(e.getMessage());
+
+        String column = columns.get(wrongIndex - 1);
+        List<String> arguments = new ArrayList<>();
+        arguments.add(header.name());
+        arguments.add(String.valueOf(lineNumber));
+        arguments.add(column);
+        arguments.add(String.valueOf(wrongIndex + 1));
+        return BundleUtil.getStringFromBundle(
+            "api.admin.datasetfield.load.ArrayIndexOutOfBoundMessage",
+            arguments
+        );
+    }
+
+    /**
+     * Get the list of columns by the type of header
+     * @param header
+     * @return
+     */
+    private List<String> getColumnsByHeader(HeaderType header) {
+        List<String> columns = null;
+        if (header.equals(HeaderType.METADATABLOCK)) {
+            columns = Arrays.asList("name", "dataverseAlias", "displayName");
+        } else if (header.equals(HeaderType.DATASETFIELD)) {
+            columns = Arrays.asList("name", "title", "description", "watermark",
+              "fieldType", "displayOrder", "displayFormat", "advancedSearchField",
+              "allowControlledVocabulary", "allowmultiples", "facetable",
+              "displayoncreate", "required", "parent", "metadatablock_id");
+        } else if (header.equals(HeaderType.CONTROLLEDVOCABULARY)) {
+            columns = Arrays.asList("DatasetField", "Value", "identifier", "displayOrder");
+        }
+
+        return columns;
+    }
+
     private String parseMetadataBlock(String[] values) {
         //Test to see if it exists by name
         MetadataBlock mdb = metadataBlockService.findByName(values[1]);

src/main/java/propertyFiles/Bundle.properties

@@ -2510,3 +2510,7 @@ externaltools.dct.displayname=Data Curation Tool
 externaltools.dct.description=Data Curation Tool for curation of variables
 externaltools.explorer.displayname=Data Explorer
 externaltools.explorer.description=The Data Explorer provides a GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis.
+
+# api/admin/datasetfield/load
+api.admin.datasetfield.load.ArrayIndexOutOfBoundMessage=Error parsing metadata block in {0} part, line #{1}: missing ''{2}'' column (#{3})
+api.admin.datasetfield.load.GeneralErrorMessage=Error parsing metadata block in {0} part, line #{1}: {2}

src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java

@@ -8,15 +8,24 @@
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.impl.GitHubOAuth2AP;
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.impl.OrcidOAuth2AP;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import static javax.ws.rs.core.Response.Status.FORBIDDEN;
 import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
 import org.junit.Test;
 import org.junit.BeforeClass;
+
+import java.util.Map;
 import java.util.UUID;
+import java.util.logging.Logger;
 
 import static javax.ws.rs.core.Response.Status.CREATED;
+import static javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR;
 import static javax.ws.rs.core.Response.Status.OK;
 import static javax.ws.rs.core.Response.Status.UNAUTHORIZED;
 import static junit.framework.Assert.assertEquals;
@@ -27,6 +36,8 @@
 
 public class AdminIT {
 
+    private static final Logger logger = Logger.getLogger(AdminIT.class.getCanonicalName());
+
     @BeforeClass
     public static void setUp() {
         RestAssured.baseURI = UtilIT.getRestAssuredBaseUri();
@@ -671,4 +682,72 @@ public void testMigrateHDLToDOI() {
                 .statusCode(OK.getStatusCode());
     }
 
+    @Test
+    public void testLoadMetadataBlock_NoErrorPath() {
+        Response createUser = UtilIT.createRandomUser();
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        byte[] updatedContent = null;
+        try {
+            updatedContent = Files.readAllBytes(Paths.get("src/test/resources/tsv/citation.tsv"));
+        } catch (IOException e) {
+            logger.warning(e.getMessage());
+            assertEquals(0,1);
+        }
+        Response response = UtilIT.loadMetadataBlock(apiToken, updatedContent);
+        assertEquals(200, response.getStatusCode());
+        response.then().assertThat().statusCode(OK.getStatusCode());
+
+        String body = response.getBody().asString();
+        String status = JsonPath.from(body).getString("status");
+        assertEquals("OK", status);
+
+        Map<String, List<Map<String, String>>> data = JsonPath.from(body).getMap("data");
+        assertEquals(1, data.size());
+        List<Map<String, String>> addedElements = data.get("added");
+        assertEquals(321, addedElements.size());
+
+        Map<String, Integer> statistics = new HashMap<>();
+        for (Map<String, String> unit : addedElements) {
+            assertEquals(2, unit.size());
+            assertTrue(unit.containsKey("name"));
+            assertTrue(unit.containsKey("type"));
+            String type = unit.get("type");
+            if (!statistics.containsKey(type))
+                statistics.put(type, 0);
+            statistics.put(type, statistics.get(type) + 1);
+        }
+
+        assertEquals(3, statistics.size());
+        assertEquals(1, (int) statistics.get("MetadataBlock"));
+        assertEquals(78, (int) statistics.get("DatasetField"));
+        assertEquals(242, (int) statistics.get("Controlled Vocabulary"));
+    }
+
+    @Test
+    public void testLoadMetadataBlock_ErrorHandling() {
+        Response createUser = UtilIT.createRandomUser();
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        byte[] updatedContent = null;
+        try {
+            updatedContent = Files.readAllBytes(Paths.get("src/test/resources/tsv/test.tsv"));
+        } catch (IOException e) {
+            logger.warning(e.getMessage());
+            assertEquals(0,1);
+        }
+        Response response = UtilIT.loadMetadataBlock(apiToken, updatedContent);
+        assertEquals(500, response.getStatusCode());
+        response.then().assertThat().statusCode(INTERNAL_SERVER_ERROR.getStatusCode());
+
+        String body = response.getBody().asString();
+        String status = JsonPath.from(body).getString("status");
+        assertEquals("ERROR", status);
+
+        String message = JsonPath.from(body).getString("message");
+        assertEquals(
+          "Error parsing metadata block in DATASETFIELD part, line #5: missing 'watermark' column (#5)",
+          message
+        );
+    }
 }

src/test/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApiTest.java

@@ -0,0 +1,62 @@
+package edu.harvard.iq.dataverse.api;
+
+import edu.harvard.iq.dataverse.util.BundleUtil;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class DatasetFieldServiceApiTest {
+
+    @Test
+    public void testArrayIndexOutOfBoundMessageBundle() {
+        List<String> arguments = new ArrayList<>();
+        arguments.add("DATASETFIELD");
+        arguments.add(String.valueOf(5));
+        arguments.add("watermark");
+        arguments.add(String.valueOf(4 + 1));
+
+        String bundle = "api.admin.datasetfield.load.ArrayIndexOutOfBoundMessage";
+        String message = BundleUtil.getStringFromBundle(bundle, arguments);
+        assertEquals(
+            "Error parsing metadata block in DATASETFIELD part, line #5: missing 'watermark' column (#5)",
+            message
+        );
+    }
+
+    @Test
+    public void testGeneralErrorMessageBundle() {
+        List<String> arguments = new ArrayList<>();
+        arguments.add("DATASETFIELD");
+        arguments.add(String.valueOf(5));
+        arguments.add("some error message");
+        String bundle = "api.admin.datasetfield.load.GeneralErrorMessage";
+        String message = BundleUtil.getStringFromBundle(bundle, arguments);
+        assertEquals(
+            "Error parsing metadata block in DATASETFIELD part, line #5: some error message",
+            message
+        );
+    }
+
+    @Test
+    public void testGetArrayIndexOutOfBoundMessage() {
+        DatasetFieldServiceApi api = new DatasetFieldServiceApi();
+        String message = api.getArrayIndexOutOfBoundMessage(DatasetFieldServiceApi.HeaderType.DATASETFIELD, 5, new ArrayIndexOutOfBoundsException("4"));
+        assertEquals(
+            "Error parsing metadata block in DATASETFIELD part, line #5: missing 'watermark' column (#5)",
+            message
+        );
+    }
+
+    @Test
+    public void testGetGeneralErrorMessage() {
+        DatasetFieldServiceApi api = new DatasetFieldServiceApi();
+        String message = api.getGeneralErrorMessage(DatasetFieldServiceApi.HeaderType.DATASETFIELD, 5, "some error");
+        assertEquals(
+            "Error parsing metadata block in DATASETFIELD part, line #5: some error",
+            message
+        );
+    }
+}

src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java

@@ -1368,6 +1368,14 @@ static Response getDatasetThumbnailMetadata(Integer datasetId, String apiToken)
                 .get("/api/admin/datasets/thumbnailMetadata/" + datasetId);
     }
 
+    static Response loadMetadataBlock(String apiToken, byte[] body) {
+        return given()
+          .header(API_TOKEN_HTTP_HEADER, apiToken)
+          .contentType("text/tab-separated-values; charset=utf-8")
+          .body(body)
+          .post("/api/admin/datasetfield/load");
+    }
+
     static Response useThumbnailFromDataFile(String datasetPersistentId, long dataFileId1, String apiToken) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)