From 8b8ee12b00647ddf954a6fb43c272bff7a95d952 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Aug 2022 19:09:16 -0400 Subject: [PATCH 1/5] 8449 switch to original --- src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index a295f264d66..00924b81da1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -260,10 +260,10 @@ public JsonObjectBuilder getOREMapBuilder(boolean aggregationOnly) throws Except if (df.getGlobalId().asString().length() != 0) { fileId = df.getGlobalId().asString(); fileSameAs = SystemConfig.getDataverseSiteUrlStatic() - + "/api/access/datafile/:persistentId?persistentId=" + fileId; + + "/api/access/datafile/:persistentId?persistentId=" + fileId + ((df.getOriginalFileFormat()!=null) ? "?format=original":""); } else { fileId = SystemConfig.getDataverseSiteUrlStatic() + "/file.xhtml?fileId=" + df.getId(); - fileSameAs = SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + df.getId(); + fileSameAs = SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + df.getId() + ((df.getOriginalFileFormat()!=null) ? "?format=original":""); } aggRes.add("@id", fileId); aggRes.add(JsonLDTerm.schemaOrg("sameAs").getLabel(), fileSameAs); From 277c84862e255daf567afdeaff02c38ca623d0f4 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Aug 2022 19:28:06 -0400 Subject: [PATCH 2/5] fix ORE file DOI case --- src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index 00924b81da1..498a3613ffa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -260,7 +260,7 @@ public JsonObjectBuilder getOREMapBuilder(boolean aggregationOnly) throws Except if (df.getGlobalId().asString().length() != 0) { fileId = df.getGlobalId().asString(); fileSameAs = SystemConfig.getDataverseSiteUrlStatic() - + "/api/access/datafile/:persistentId?persistentId=" + fileId + ((df.getOriginalFileFormat()!=null) ? "?format=original":""); + + "/api/access/datafile/:persistentId?persistentId=" + fileId + ((df.getOriginalFileFormat()!=null) ? "&format=original":""); } else { fileId = SystemConfig.getDataverseSiteUrlStatic() + "/file.xhtml?fileId=" + df.getId(); fileSameAs = SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + df.getId() + ((df.getOriginalFileFormat()!=null) ? "?format=original":""); From 47c570ca764a1a7d39b9d61943018ba63462af43 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Sat, 6 Aug 2022 13:03:09 -0400 Subject: [PATCH 3/5] use original name/mimetype/size if ingested --- .../iq/dataverse/util/bagit/OREMap.java | 38 +++++++++++++++---- .../iq/dataverse/util/json/JsonLDTerm.java | 3 ++ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index 498a3613ffa..048716ce505 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -237,7 +237,32 @@ public JsonObjectBuilder getOREMapBuilder(boolean aggregationOnly) throws Except } else { addIfNotNull(aggRes, JsonLDTerm.schemaOrg("description"), df.getDescription()); } - addIfNotNull(aggRes, JsonLDTerm.schemaOrg("name"), fmd.getLabel()); // "label" is the filename + String fileName = fmd.getLabel();// "label" is the filename + long fileSize = df.getFilesize(); + String mimeType = df.getContentType(); + String currentIngestedName = null; + boolean ingested=df.getOriginalFileName()!= null || df.getOriginalFileSize()!=null || df.getOriginalFileFormat()!=null; + if(ingested) { + if(df.getOriginalFileName()!=null) { + currentIngestedName= fileName; + fileName = df.getOriginalFileName(); + } else { + logger.warning("Missing Original file name for id: " + df.getId()); + } + if(df.getOriginalFileSize()!=null) { + fileSize = df.getOriginalFileSize(); + } else { + logger.warning("Missing Original file size for id: " + df.getId()); + } + if(df.getOriginalFileFormat()!=null) { + mimeType = df.getOriginalFileFormat(); + } else { + logger.warning("Missing Original file format for id: " + df.getId()); + } + + + } + addIfNotNull(aggRes, JsonLDTerm.schemaOrg("name"), fileName); addIfNotNull(aggRes, JsonLDTerm.restricted, fmd.isRestricted()); addIfNotNull(aggRes, JsonLDTerm.directoryLabel, fmd.getDirectoryLabel()); addIfNotNull(aggRes, JsonLDTerm.schemaOrg("version"), fmd.getVersion()); @@ -260,21 +285,20 @@ public JsonObjectBuilder getOREMapBuilder(boolean aggregationOnly) throws Except if (df.getGlobalId().asString().length() != 0) { fileId = df.getGlobalId().asString(); fileSameAs = SystemConfig.getDataverseSiteUrlStatic() - + "/api/access/datafile/:persistentId?persistentId=" + fileId + ((df.getOriginalFileFormat()!=null) ? "&format=original":""); + + "/api/access/datafile/:persistentId?persistentId=" + fileId + (ingested ? "&format=original":""); } else { fileId = SystemConfig.getDataverseSiteUrlStatic() + "/file.xhtml?fileId=" + df.getId(); - fileSameAs = SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + df.getId() + ((df.getOriginalFileFormat()!=null) ? "?format=original":""); + fileSameAs = SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + df.getId() + (ingested ? "?format=original":""); } aggRes.add("@id", fileId); aggRes.add(JsonLDTerm.schemaOrg("sameAs").getLabel(), fileSameAs); fileArray.add(fileId); aggRes.add("@type", JsonLDTerm.ore("AggregatedResource").getLabel()); - addIfNotNull(aggRes, JsonLDTerm.schemaOrg("fileFormat"), df.getContentType()); - addIfNotNull(aggRes, JsonLDTerm.filesize, df.getFilesize()); + addIfNotNull(aggRes, JsonLDTerm.schemaOrg("fileFormat"), mimeType); + addIfNotNull(aggRes, JsonLDTerm.filesize, fileSize); addIfNotNull(aggRes, JsonLDTerm.storageIdentifier, df.getStorageIdentifier()); - addIfNotNull(aggRes, JsonLDTerm.originalFileFormat, df.getOriginalFileFormat()); - addIfNotNull(aggRes, JsonLDTerm.originalFormatLabel, df.getOriginalFormatLabel()); + addIfNotNull(aggRes, JsonLDTerm.currentIngestedName, currentIngestedName); addIfNotNull(aggRes, JsonLDTerm.UNF, df.getUnf()); addIfNotNull(aggRes, JsonLDTerm.rootDataFileId, df.getRootDataFileId()); addIfNotNull(aggRes, JsonLDTerm.previousDataFileId, df.getPreviousDataFileId()); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDTerm.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDTerm.java index 20aeceda7de..065097709cf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDTerm.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonLDTerm.java @@ -34,8 +34,11 @@ public class JsonLDTerm { public static JsonLDTerm categories = JsonLDTerm.DVCore("categories"); public static JsonLDTerm filesize = JsonLDTerm.DVCore("filesize"); public static JsonLDTerm storageIdentifier = JsonLDTerm.DVCore("storageIdentifier"); + @Deprecated public static JsonLDTerm originalFileFormat = JsonLDTerm.DVCore("originalFileFormat"); + @Deprecated public static JsonLDTerm originalFormatLabel = JsonLDTerm.DVCore("originalFormatLabel"); + public static JsonLDTerm currentIngestedName= JsonLDTerm.DVCore("currentIngestedName"); public static JsonLDTerm UNF = JsonLDTerm.DVCore("UNF"); public static JsonLDTerm rootDataFileId = JsonLDTerm.DVCore("rootDataFileId"); public static JsonLDTerm previousDataFileId = JsonLDTerm.DVCore("previousDataFileId"); From bf351155bcc98e4a04a9cf40219329b9034db785 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 9 Aug 2022 16:36:38 -0400 Subject: [PATCH 4/5] add test to (locally) exercise submitDatasetVersionToArchive #8449 --- .../edu/harvard/iq/dataverse/api/BagIT.java | 47 +++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 14 ++++++ 2 files changed, 61 insertions(+) create mode 100644 src/test/java/edu/harvard/iq/dataverse/api/BagIT.java diff --git a/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java b/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java new file mode 100644 index 00000000000..29c6f45ec22 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java @@ -0,0 +1,47 @@ +package edu.harvard.iq.dataverse.api; + +import com.jayway.restassured.response.Response; +import static javax.ws.rs.core.Response.Status.CREATED; +import static javax.ws.rs.core.Response.Status.OK; +import org.junit.Test; + +public class BagIT { + + /** + * Note: to run this test you have to configure an archiver, such as the + * local archiver. + */ + @Test + public void testBagItExport() { + + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + Response toggleSuperuser = UtilIT.makeSuperUser(username); + toggleSuperuser.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response createDataverse = UtilIT.createRandomDataverse(apiToken); + createDataverse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + Integer dataverseId = UtilIT.getDataverseIdFromResponse(createDataverse); + + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + String datasetPid = UtilIT.getDatasetPersistentIdFromResponse(createDataset); + + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); + publishDataverse.then().assertThat().statusCode(OK.getStatusCode()); + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPid, "major", apiToken); + publishDataset.then().assertThat().statusCode(OK.getStatusCode()); + + Response archiveDataset = UtilIT.archiveDataset(datasetPid, "1.0", apiToken); + archiveDataset.prettyPrint(); + archiveDataset.then().assertThat().statusCode(OK.getStatusCode()); + + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index c791ce72f41..a029f7446f2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2917,6 +2917,20 @@ static Response getDatasetVersionArchivalStatus(Integer datasetId, String versio .get("/api/datasets/" + datasetId + "/" + version + "/archivalStatus"); return response; } + + static Response archiveDataset(String idOrPersistentIdOfDataset, String version, String apiToken) { + String idInPath = idOrPersistentIdOfDataset; + String optionalQueryParam = ""; + if (!NumberUtils.isNumber(idOrPersistentIdOfDataset)) { + idInPath = ":persistentId"; + optionalQueryParam = "?persistentId=" + idOrPersistentIdOfDataset; + } + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .post("/api/admin/submitDatasetVersionToArchive/" + idInPath + "/" + version + optionalQueryParam); + return response; + } + static Response setDatasetVersionArchivalStatus(Integer datasetId, String version, String apiToken, String status, String message) { Response response = given() .header(API_TOKEN_HTTP_HEADER, apiToken).contentType("application/json; charset=utf-8").body("{\"status\":\"" + status + "\", \"message\":\"" + message + "\"}") From 6210a71670af328fd927331ce7cb07fa5e54781e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 10 Aug 2022 11:53:48 -0400 Subject: [PATCH 5/5] add BagIT to API test suite #8449 Also allow strings as setting keys because :BagItLocalPath isn't in the usual enum. --- .../edu/harvard/iq/dataverse/api/BagIT.java | 39 +++++++++++++++++-- .../edu/harvard/iq/dataverse/api/UtilIT.java | 16 ++++++++ tests/integration-tests.txt | 2 +- 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java b/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java index 29c6f45ec22..4ac76ac846d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java @@ -1,16 +1,36 @@ package edu.harvard.iq.dataverse.api; +import com.jayway.restassured.RestAssured; import com.jayway.restassured.response.Response; +import edu.harvard.iq.dataverse.engine.command.impl.LocalSubmitToArchiveCommand; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import static javax.ws.rs.core.Response.Status.CREATED; import static javax.ws.rs.core.Response.Status.OK; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; public class BagIT { - /** - * Note: to run this test you have to configure an archiver, such as the - * local archiver. - */ + @BeforeClass + public static void setUpClass() { + + RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); + + Response setArchiverClassName = UtilIT.setSetting(SettingsServiceBean.Key.ArchiverClassName, LocalSubmitToArchiveCommand.class.getCanonicalName()); + setArchiverClassName.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response setArchiverSettings = UtilIT.setSetting(SettingsServiceBean.Key.ArchiverSettings, ":BagItLocalPath, :BagGeneratorThreads"); + setArchiverSettings.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response setBagItLocalPath = UtilIT.setSetting(":BagItLocalPath", "/tmp"); + setBagItLocalPath.then().assertThat() + .statusCode(OK.getStatusCode()); + + } + @Test public void testBagItExport() { @@ -44,4 +64,15 @@ public void testBagItExport() { archiveDataset.then().assertThat().statusCode(OK.getStatusCode()); } + + @AfterClass + public static void tearDownClass() { + + // Not checking if delete happened. Hopefully, it did. + UtilIT.deleteSetting(SettingsServiceBean.Key.ArchiverClassName); + UtilIT.deleteSetting(SettingsServiceBean.Key.ArchiverSettings); + UtilIT.deleteSetting(":BagItLocalPath"); + + } + } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index a029f7446f2..03a0e47bc71 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1869,6 +1869,14 @@ static Response deleteSetting(SettingsServiceBean.Key settingKey) { return response; } + /** + * @param settingKey Include the colon like :BagItLocalPath + */ + static Response deleteSetting(String settingKey) { + Response response = given().when().delete("/api/admin/settings/" + settingKey); + return response; + } + static Response getSetting(SettingsServiceBean.Key settingKey) { Response response = given().when().get("/api/admin/settings/" + settingKey); return response; @@ -1879,6 +1887,14 @@ static Response setSetting(SettingsServiceBean.Key settingKey, String value) { return response; } + /** + * @param settingKey Include the colon like :BagItLocalPath + */ + public static Response setSetting(String settingKey, String value) { + Response response = given().body(value).when().put("/api/admin/settings/" + settingKey); + return response; + } + static Response getRoleAssignmentsOnDataverse(String dataverseAliasOrId, String apiToken) { String url = "/api/dataverses/" + dataverseAliasOrId + "/assignments"; return given() diff --git a/tests/integration-tests.txt b/tests/integration-tests.txt index 71ba38e0aae..85b37c79835 100644 --- a/tests/integration-tests.txt +++ b/tests/integration-tests.txt @@ -1 +1 @@ -DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT +DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT