From 8cfab9d855f2e54a36163146ae14cdf7b8186a38 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 14 Jul 2023 11:55:21 -0400 Subject: [PATCH 1/4] restore datatable/var metadata/etc to file details export --- .../edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 97640aa226d..2e098aa928b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -698,7 +698,12 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo * InternalExportDataProvider fileDetails. */ if (forExportDataProvider) { - builder.add("restricted", df.isRestricted()); + builder.add("restricted", df.isRestricted()) + .add("fileMetadataId", fileMetadata.getId()) + .add("dataTables", df.getDataTables().isEmpty() ? null : JsonPrinter.jsonDT(df.getDataTables())) + .add("varGroups", fileMetadata.getVarGroups().isEmpty() + ? JsonPrinter.jsonVarGroup(fileMetadata.getVarGroups()) + : null); } return builder; } From 8763b86d8d14c5e8832b2ef7d6e96020bedab993 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 17 Jul 2023 09:44:51 -0400 Subject: [PATCH 2/4] Add test for unrestricted file ddi export --- .../harvard/iq/dataverse/api/DatasetsIT.java | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 687ab453d24..6ceeccfe34a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -2276,6 +2276,71 @@ public void testLinkingDatasets() { */ } + /** + * This tests the "DDI export" and verifies that variable metadata is included for an unrestricted file. + */ + @Test + public void testUnrestrictedFileExportDdi() throws IOException { + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String authorUsername = UtilIT.getUsernameFromResponse(createUser); + String authorApiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverse = UtilIT.createRandomDataverse(authorApiToken); + createDataverse.prettyPrint(); + createDataverse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, authorApiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + String datasetPid = JsonPath.from(createDataset.asString()).getString("data.persistentId"); + + Path pathToFile = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "data.csv"); + String contentOfCsv = "" + + "name,pounds,species\n" + + "Marshall,40,dog\n" + + "Tiger,17,cat\n" + + "Panther,21,cat\n"; + java.nio.file.Files.write(pathToFile, contentOfCsv.getBytes()); + + Response uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile.toString(), authorApiToken); + uploadFile.prettyPrint(); + uploadFile.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.files[0].label", equalTo("data.csv")); + + String fileId = JsonPath.from(uploadFile.body().asString()).getString("data.files[0].dataFile.id"); + + assertTrue("Failed test if Ingest Lock exceeds max duration " + pathToFile, UtilIT.sleepForLock(datasetId.longValue(), "Ingest", authorApiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION)); + + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, authorApiToken); + publishDataverse.then().assertThat().statusCode(OK.getStatusCode()); + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPid, "major", authorApiToken); + publishDataset.then().assertThat().statusCode(OK.getStatusCode()); + + // We're testing export here, which is at dataset level. + // Guest/public version + Response exportByGuest = UtilIT.exportDataset(datasetPid, "ddi"); + exportByGuest.prettyPrint(); + exportByGuest.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("codeBook.fileDscr.fileTxt.fileName", equalTo("data.tab")) + .body("codeBook.fileDscr.dimensns.caseQnty", equalTo("3")) + .body("codeBook.fileDscr.dimensns.varQnty", equalTo("3")) + .body("codeBook.dataDscr", CoreMatchers.not(equalTo(null))) + .body("codeBook.dataDscr.var[0].@name", equalTo("name")) + .body("codeBook.dataDscr.var[1].@name", equalTo("pounds")) + // This is an example of a summary stat (max) that should be visible. + .body("codeBook.dataDscr.var[1].sumStat.find { it.@type == 'max' }", equalTo("40.0")) + .body("codeBook.dataDscr.var[2].@name", equalTo("species")); + } + /** * In this test we are restricting a file and testing "export DDI" at the * dataset level as well as getting the DDI at the file level. From 5ad954b71b6c5464a4a7ba3ff6768b0c68d6e0bc Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 17 Jul 2023 10:31:20 -0400 Subject: [PATCH 3/4] path typo --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 6ceeccfe34a..866524a2605 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -2331,8 +2331,8 @@ public void testUnrestrictedFileExportDdi() throws IOException { exportByGuest.then().assertThat() .statusCode(OK.getStatusCode()) .body("codeBook.fileDscr.fileTxt.fileName", equalTo("data.tab")) - .body("codeBook.fileDscr.dimensns.caseQnty", equalTo("3")) - .body("codeBook.fileDscr.dimensns.varQnty", equalTo("3")) + .body("codeBook.fileDscr.fileTxt.dimensns.caseQnty", equalTo("3")) + .body("codeBook.fileDscr.fileTxt.dimensns.varQnty", equalTo("3")) .body("codeBook.dataDscr", CoreMatchers.not(equalTo(null))) .body("codeBook.dataDscr.var[0].@name", equalTo("name")) .body("codeBook.dataDscr.var[1].@name", equalTo("pounds")) From 7911b9d3fc07126284cf92d72419ff011036fa52 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 17 Jul 2023 12:07:59 -0400 Subject: [PATCH 4/4] avoid quotes in sumStat outputs --- .../edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 73ba2d204ce..8f2aa5d023a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -56,6 +56,7 @@ import javax.json.JsonArray; import javax.json.JsonArrayBuilder; import javax.json.JsonObject; +import javax.json.JsonString; import javax.json.JsonValue; import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; @@ -1917,7 +1918,7 @@ private static void createVarDDI(XMLStreamWriter xmlw, JsonObject dvar, String f for (Entry sumStat : dvar.getJsonObject("summaryStatistics").entrySet()) { xmlw.writeStartElement("sumStat"); writeAttribute(xmlw, "type", sumStat.getKey()); - xmlw.writeCharacters(sumStat.getValue().toString()); + xmlw.writeCharacters(((JsonString)sumStat.getValue()).getString()); xmlw.writeEndElement(); // sumStat } }