diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml
index e36a78b11be..bf37299f2df 100644
--- a/modules/dataverse-parent/pom.xml
+++ b/modules/dataverse-parent/pom.xml
@@ -163,7 +163,7 @@
4.4.14
- 5.0.0-RC1
+ 5.0.0-RC2
1.15.0
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java
index 42534514b68..b75cb687c62 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java
@@ -373,13 +373,13 @@ public Response startHarvestingJob(@PathParam("nickName") String clientNickname,
}
if (authenticatedUser == null || !authenticatedUser.isSuperuser()) {
- return error(Response.Status.FORBIDDEN, "Only the Dataverse Admin user can run harvesting jobs");
+ return error(Response.Status.FORBIDDEN, "Only admin users can run harvesting jobs");
}
HarvestingClient harvestingClient = harvestingClientService.findByNickname(clientNickname);
if (harvestingClient == null) {
- return error(Response.Status.NOT_FOUND, "No such dataverse: "+clientNickname);
+ return error(Response.Status.NOT_FOUND, "No such client: "+clientNickname);
}
DataverseRequest dataverseRequest = createDataverseRequest(authenticatedUser);
@@ -391,35 +391,8 @@ public Response startHarvestingJob(@PathParam("nickName") String clientNickname,
return this.accepted();
}
- // This GET shows the status of the harvesting run in progress for this
- // client, if present:
- // @GET
- // @Path("{nickName}/run")
- // TODO:
-
- // This DELETE kills the harvesting run in progress for this client,
- // if present:
- // @DELETE
- // @Path("{nickName}/run")
- // TODO:
-
-
-
-
-
/* Auxiliary, helper methods: */
- /*
- @Deprecated
- public static JsonArrayBuilder harvestingConfigsAsJsonArray(List harvestingDataverses) {
- JsonArrayBuilder hdArr = Json.createArrayBuilder();
-
- for (Dataverse hd : harvestingDataverses) {
- hdArr.add(harvestingConfigAsJson(hd.getHarvestingClientConfig()));
- }
- return hdArr;
- }*/
-
public static JsonObjectBuilder harvestingConfigAsJson(HarvestingClient harvestingConfig) {
if (harvestingConfig == null) {
return null;
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java
index 5b3e4df331d..c5e3a93e2df 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java
@@ -130,7 +130,7 @@ public void harvestRecord(String baseURL, String identifier, String metadataPref
int responseCode = 0;
con = (HttpURLConnection) url.openConnection();
- con.setRequestProperty("User-Agent", "DataverseHarvester/3.0");
+ con.setRequestProperty("User-Agent", "Dataverse Harvesting Client v5");
con.setRequestProperty("Accept-Encoding",
"compress, gzip, identify");
try {
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
index 058a20451d6..0e9ffb20653 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
@@ -360,7 +360,7 @@ File retrieveProprietaryDataverseMetadata (HttpClient client, String remoteApiUr
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(remoteApiUrl))
.GET()
- .header("User-Agent", "DataverseHarvester/6.0")
+ .header("User-Agent", "Dataverse Harvesting Client v5")
.build();
HttpResponse response;
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java
index faf3cf9ddc4..147d42648fa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java
@@ -49,7 +49,7 @@ public DataverseXoaiItemRepository (OAIRecordServiceBean recordService, DatasetS
}
@Override
- public ItemIdentifier getItem(String identifier) throws IdDoesNotExistException {
+ public ItemIdentifier getItemIdentifier(String identifier) throws IdDoesNotExistException {
// This method is called when ListMetadataFormats request specifies
// the identifier, requesting the formats available for this specific record.
// In our case, under the current implementation, we need to simply look
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java
index 9eac3545e54..094eb0df77c 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java
@@ -1,34 +1,59 @@
package edu.harvard.iq.dataverse.api;
import java.util.logging.Logger;
+import java.util.logging.Level;
import com.jayway.restassured.RestAssured;
import static com.jayway.restassured.RestAssured.given;
+import com.jayway.restassured.path.json.JsonPath;
import org.junit.Test;
import com.jayway.restassured.response.Response;
+import static javax.ws.rs.core.Response.Status.CREATED;
+import static javax.ws.rs.core.Response.Status.UNAUTHORIZED;
+import static javax.ws.rs.core.Response.Status.ACCEPTED;
+import static javax.ws.rs.core.Response.Status.OK;
import static org.hamcrest.CoreMatchers.equalTo;
-import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import org.junit.BeforeClass;
/**
- * extremely minimal (for now) API tests for creating OAI clients.
+ * This class tests Harvesting Client functionality.
+ * Note that these methods test BOTH the proprietary Dataverse rest API for
+ * creating and managing harvesting clients, AND the underlining OAI-PMH
+ * harvesting functionality itself. I.e., we will use the Dataverse
+ * /api/harvest/clients/ api to run an actual harvest of a control set and
+ * then validate the resulting harvested content.
*/
public class HarvestingClientsIT {
private static final Logger logger = Logger.getLogger(HarvestingClientsIT.class.getCanonicalName());
- private static final String harvestClientsApi = "/api/harvest/clients/";
- private static final String harvestCollection = "root";
- private static final String harvestUrl = "https://demo.dataverse.org/oai";
- private static final String archiveUrl = "https://demo.dataverse.org";
- private static final String harvestMetadataFormat = "oai_dc";
- private static final String archiveDescription = "RestAssured harvesting client test";
+ private static final String HARVEST_CLIENTS_API = "/api/harvest/clients/";
+ private static final String ROOT_COLLECTION = "root";
+ private static final String HARVEST_URL = "https://demo.dataverse.org/oai";
+ private static final String ARCHIVE_URL = "https://demo.dataverse.org";
+ private static final String HARVEST_METADATA_FORMAT = "oai_dc";
+ private static final String ARCHIVE_DESCRIPTION = "RestAssured harvesting client test";
+ private static final String CONTROL_OAI_SET = "controlTestSet";
+ private static final int DATASETS_IN_CONTROL_SET = 7;
+ private static String normalUserAPIKey;
+ private static String adminUserAPIKey;
+ private static String harvestCollectionAlias;
@BeforeClass
public static void setUpClass() {
RestAssured.baseURI = UtilIT.getRestAssuredBaseUri();
+
+ // Create the users, an admin and a non-admin:
+ setupUsers();
+
+ // Create a collection that we will use to harvest remote content into:
+ setupCollection();
+
}
- private void setupUsers() {
+ private static void setupUsers() {
Response cu0 = UtilIT.createRandomUser();
normalUserAPIKey = UtilIT.getApiTokenFromResponse(cu0);
Response cu1 = UtilIT.createRandomUser();
@@ -36,23 +61,34 @@ private void setupUsers() {
Response u1a = UtilIT.makeSuperUser(un1);
adminUserAPIKey = UtilIT.getApiTokenFromResponse(cu1);
}
+
+ private static void setupCollection() {
+ Response createDataverseResponse = UtilIT.createRandomDataverse(adminUserAPIKey);
+ createDataverseResponse.prettyPrint();
+ assertEquals(CREATED.getStatusCode(), createDataverseResponse.getStatusCode());
+
+ harvestCollectionAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
- private String normalUserAPIKey;
- private String adminUserAPIKey;
+ // publish dataverse:
+ Response publishDataverse = UtilIT.publishDataverseViaNativeApi(harvestCollectionAlias, adminUserAPIKey);
+ assertEquals(OK.getStatusCode(), publishDataverse.getStatusCode());
+ }
@Test
public void testCreateEditDeleteClient() {
- setupUsers();
+ // This method focuses on testing the native Dataverse harvesting client
+ // API.
+
String nickName = UtilIT.getRandomString(6);
- String clientApiPath = String.format(harvestClientsApi+"%s", nickName);
+ String clientApiPath = String.format(HARVEST_CLIENTS_API+"%s", nickName);
String clientJson = String.format("{\"dataverseAlias\":\"%s\","
+ "\"type\":\"oai\","
+ "\"harvestUrl\":\"%s\","
+ "\"archiveUrl\":\"%s\","
+ "\"metadataFormat\":\"%s\"}",
- harvestCollection, harvestUrl, archiveUrl, harvestMetadataFormat);
+ ROOT_COLLECTION, HARVEST_URL, ARCHIVE_URL, HARVEST_METADATA_FORMAT);
// Try to create a client as normal user, should fail:
@@ -61,7 +97,7 @@ public void testCreateEditDeleteClient() {
.header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey)
.body(clientJson)
.post(clientApiPath);
- assertEquals(401, rCreate.getStatusCode());
+ assertEquals(UNAUTHORIZED.getStatusCode(), rCreate.getStatusCode());
// Try to create the same as admin user, should succeed:
@@ -70,17 +106,17 @@ public void testCreateEditDeleteClient() {
.header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey)
.body(clientJson)
.post(clientApiPath);
- assertEquals(201, rCreate.getStatusCode());
+ assertEquals(CREATED.getStatusCode(), rCreate.getStatusCode());
// Try to update the client we have just created:
- String updateJson = String.format("{\"archiveDescription\":\"%s\"}", archiveDescription);
+ String updateJson = String.format("{\"archiveDescription\":\"%s\"}", ARCHIVE_DESCRIPTION);
Response rUpdate = given()
.header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey)
.body(updateJson)
.put(clientApiPath);
- assertEquals(200, rUpdate.getStatusCode());
+ assertEquals(OK.getStatusCode(), rUpdate.getStatusCode());
// Now let's retrieve the client we've just created and edited:
@@ -89,7 +125,7 @@ public void testCreateEditDeleteClient() {
logger.info("getClient.getStatusCode(): " + getClientResponse.getStatusCode());
logger.info("getClient printresponse: " + getClientResponse.prettyPrint());
- assertEquals(200, getClientResponse.getStatusCode());
+ assertEquals(OK.getStatusCode(), getClientResponse.getStatusCode());
// ... and validate the values:
@@ -97,11 +133,11 @@ public void testCreateEditDeleteClient() {
.body("status", equalTo(AbstractApiBean.STATUS_OK))
.body("data.type", equalTo("oai"))
.body("data.nickName", equalTo(nickName))
- .body("data.archiveDescription", equalTo(archiveDescription))
- .body("data.dataverseAlias", equalTo(harvestCollection))
- .body("data.harvestUrl", equalTo(harvestUrl))
- .body("data.archiveUrl", equalTo(archiveUrl))
- .body("data.metadataFormat", equalTo(harvestMetadataFormat));
+ .body("data.archiveDescription", equalTo(ARCHIVE_DESCRIPTION))
+ .body("data.dataverseAlias", equalTo(ROOT_COLLECTION))
+ .body("data.harvestUrl", equalTo(HARVEST_URL))
+ .body("data.archiveUrl", equalTo(ARCHIVE_URL))
+ .body("data.metadataFormat", equalTo(HARVEST_METADATA_FORMAT));
// Try to delete the client as normal user should fail:
@@ -109,7 +145,7 @@ public void testCreateEditDeleteClient() {
.header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey)
.delete(clientApiPath);
logger.info("rDelete.getStatusCode(): " + rDelete.getStatusCode());
- assertEquals(401, rDelete.getStatusCode());
+ assertEquals(UNAUTHORIZED.getStatusCode(), rDelete.getStatusCode());
// Try to delete as admin user should work:
@@ -117,6 +153,118 @@ public void testCreateEditDeleteClient() {
.header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey)
.delete(clientApiPath);
logger.info("rDelete.getStatusCode(): " + rDelete.getStatusCode());
- assertEquals(200, rDelete.getStatusCode());
+ assertEquals(OK.getStatusCode(), rDelete.getStatusCode());
+ }
+
+ @Test
+ public void testHarvestingClientRun() throws InterruptedException {
+ // This test will create a client and attempt to perform an actual
+ // harvest and validate the resulting harvested content.
+
+ // Setup: create the client via native API
+ // since this API is tested somewhat extensively in the previous
+ // method, we don't need to pay too much attention to this method, aside
+ // from confirming the expected HTTP status code.
+
+ String nickName = UtilIT.getRandomString(6);
+
+ String clientApiPath = String.format(HARVEST_CLIENTS_API+"%s", nickName);
+ String clientJson = String.format("{\"dataverseAlias\":\"%s\","
+ + "\"type\":\"oai\","
+ + "\"harvestUrl\":\"%s\","
+ + "\"archiveUrl\":\"%s\","
+ + "\"set\":\"%s\","
+ + "\"metadataFormat\":\"%s\"}",
+ harvestCollectionAlias, HARVEST_URL, ARCHIVE_URL, CONTROL_OAI_SET, HARVEST_METADATA_FORMAT);
+
+ Response createResponse = given()
+ .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey)
+ .body(clientJson)
+ .post(clientApiPath);
+ assertEquals(CREATED.getStatusCode(), createResponse.getStatusCode());
+
+ // API TEST 1. Run the harvest using the configuration (client) we have
+ // just created
+
+ String runHarvestApiPath = String.format(HARVEST_CLIENTS_API+"%s/run", nickName);
+
+ // TODO? - verify that a non-admin user cannot perform this operation (401)
+
+ Response runResponse = given()
+ .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey)
+ .post(runHarvestApiPath);
+ assertEquals(ACCEPTED.getStatusCode(), runResponse.getStatusCode());
+
+ // API TEST 2. As indicated by the ACCEPTED status code above, harvesting
+ // is an asynchronous operation that will be performed in the background.
+ // Verify that this "in progress" status is properly reported while it's
+ // running, and that it completes in some reasonable amount of time.
+
+ int i = 0;
+ int maxWait=20; // a very conservative interval; this harvest has no business taking this long
+ do {
+ // Give it an initial 1 sec. delay, to make sure the client state
+ // has been updated in the database, which can take some appreciable
+ // amount of time on a heavily-loaded server running a full suite of
+ // tests:
+ Thread.sleep(1000L);
+ // keep checking the status of the client with the GET api:
+ Response getClientResponse = given()
+ .get(clientApiPath);
+
+ assertEquals(OK.getStatusCode(), getClientResponse.getStatusCode());
+ JsonPath responseJsonPath = getClientResponse.body().jsonPath();
+ assertNotNull("Invalid JSON in GET client response", responseJsonPath);
+ assertEquals(AbstractApiBean.STATUS_OK, responseJsonPath.getString("status"));
+
+ String clientStatus = responseJsonPath.getString("data.status");
+ assertNotNull(clientStatus);
+
+ if ("inProgress".equals(clientStatus) || "IN PROGRESS".equals(responseJsonPath.getString("data.lastResult"))) {
+ // we'll sleep for another second
+ i++;
+ } else {
+ logger.info("getClientResponse.prettyPrint: "
+ + getClientResponse.prettyPrint());
+ // Check the values in the response:
+ // a) Confirm that the harvest has completed:
+ assertEquals("Unexpected client status: "+clientStatus, "inActive", clientStatus);
+
+ // b) Confirm that it has actually succeeded:
+ assertEquals("Last harvest not reported a success (took "+i+" seconds)", "SUCCESS", responseJsonPath.getString("data.lastResult"));
+ String harvestTimeStamp = responseJsonPath.getString("data.lastHarvest");
+ assertNotNull(harvestTimeStamp);
+
+ // c) Confirm that the other timestamps match:
+ assertEquals(harvestTimeStamp, responseJsonPath.getString("data.lastSuccessful"));
+ assertEquals(harvestTimeStamp, responseJsonPath.getString("data.lastNonEmpty"));
+
+ // d) Confirm that the correct number of datasets have been harvested:
+ assertEquals(DATASETS_IN_CONTROL_SET, responseJsonPath.getInt("data.lastDatasetsHarvested"));
+
+ // ok, it looks like the harvest has completed successfully.
+ break;
+ }
+ } while (i extraDatasetsIdentifiers = new ArrayList<>();
+
@BeforeClass
public static void setUpClass() {
RestAssured.baseURI = UtilIT.getRestAssuredBaseUri();
// enable harvesting server
// Gave some thought to storing the original response, and resetting afterwards - but that appears to be more complexity than it's worth
Response enableHarvestingServerResponse = UtilIT.setSetting(SettingsServiceBean.Key.OAIServerEnabled,"true");
+
+ // Create users:
+ setupUsers();
+
+ // Create and publish some datasets:
+ setupDatasets();
+
}
@AfterClass
@@ -44,7 +59,7 @@ public static void afterClass() {
Response enableHarvestingServerResponse = UtilIT.setSetting(SettingsServiceBean.Key.OAIServerEnabled,"false");
}
- private void setupUsers() {
+ private static void setupUsers() {
Response cu0 = UtilIT.createRandomUser();
normalUserAPIKey = UtilIT.getApiTokenFromResponse(cu0);
Response cu1 = UtilIT.createRandomUser();
@@ -52,6 +67,62 @@ private void setupUsers() {
Response u1a = UtilIT.makeSuperUser(un1);
adminUserAPIKey = UtilIT.getApiTokenFromResponse(cu1);
}
+
+ private static void setupDatasets() {
+ // create dataverse:
+ Response createDataverseResponse = UtilIT.createRandomDataverse(adminUserAPIKey);
+ createDataverseResponse.prettyPrint();
+ String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+ // publish dataverse:
+ Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, adminUserAPIKey);
+ assertEquals(OK.getStatusCode(), publishDataverse.getStatusCode());
+
+ // create dataset:
+ Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, adminUserAPIKey);
+ createDatasetResponse.prettyPrint();
+ Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse);
+
+ // retrieve the global id:
+ singleSetDatasetPersistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse);
+
+ // publish dataset:
+ Response publishDataset = UtilIT.publishDatasetViaNativeApi(singleSetDatasetPersistentId, "major", adminUserAPIKey);
+ assertEquals(200, publishDataset.getStatusCode());
+
+ singleSetDatasetIdentifier = singleSetDatasetPersistentId.substring(singleSetDatasetPersistentId.lastIndexOf('/') + 1);
+
+ logger.info("identifier: " + singleSetDatasetIdentifier);
+
+ // Publish command is executed asynchronously, i.e. it may
+ // still be running after we received the OK from the publish API.
+ // The oaiExport step also requires the metadata exports to be done and this
+ // takes longer than just publish/reindex.
+ // So wait for all of this to finish.
+ UtilIT.sleepForReexport(singleSetDatasetPersistentId, adminUserAPIKey, 10);
+
+ // ... And let's create 4 more datasets for a multi-dataset experiment:
+
+ for (int i = 0; i < 4; i++) {
+ // create dataset:
+ createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, adminUserAPIKey);
+ createDatasetResponse.prettyPrint();
+ datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse);
+
+ // retrieve the global id:
+ String thisDatasetPersistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse);
+
+ // publish dataset:
+ publishDataset = UtilIT.publishDatasetViaNativeApi(thisDatasetPersistentId, "major", adminUserAPIKey);
+ assertEquals(200, publishDataset.getStatusCode());
+
+ UtilIT.sleepForReexport(thisDatasetPersistentId, adminUserAPIKey, 10);
+
+ extraDatasetsIdentifiers.add(thisDatasetPersistentId.substring(thisDatasetPersistentId.lastIndexOf('/') + 1));
+ }
+
+
+ }
private String jsonForTestSpec(String name, String def) {
String r = String.format("{\"name\":\"%s\",\"definition\":\"%s\"}", name, def);//description is optional
@@ -63,178 +134,271 @@ private String jsonForEditSpec(String name, String def, String desc) {
return r;
}
- private String normalUserAPIKey;
- private String adminUserAPIKey;
+ private XmlPath validateOaiVerbResponse(Response oaiResponse, String verb) {
+ logger.info(verb+" response: "+oaiResponse.prettyPrint());
+ // confirm that the response is in fact XML:
+ XmlPath responseXmlPath = oaiResponse.getBody().xmlPath();
+ assertNotNull(responseXmlPath);
+
+ String dateString = responseXmlPath.getString("OAI-PMH.responseDate");
+ assertNotNull(dateString);
+ // TODO: validate the formatting of the date string in the record
+ // header, above. (could be slightly tricky - since this formatting
+ // is likely locale-specific)
+ logger.fine("date string from the OAI output:"+dateString);
+ //assertEquals("http://localhost:8080/oai", responseXmlPath.getString("OAI-PMH.request"));
+ assertEquals(verb, responseXmlPath.getString("OAI-PMH.request.@verb"));
+ return responseXmlPath;
+ }
+
+ @Test
+ public void testOaiIdentify() {
+ // Run Identify:
+ Response identifyResponse = UtilIT.getOaiIdentify();
+ assertEquals(OK.getStatusCode(), identifyResponse.getStatusCode());
+ // Validate the response:
+
+ XmlPath responseXmlPath = validateOaiVerbResponse(identifyResponse, "Identify");
+ //assertEquals("http://localhost:8080/oai", responseXmlPath.getString("OAI-PMH.Identify.baseURL"));
+ // Confirm that the server is reporting the correct parameters that
+ // our server implementation should be using:
+ assertEquals("2.0", responseXmlPath.getString("OAI-PMH.Identify.protocolVersion"));
+ assertEquals("transient", responseXmlPath.getString("OAI-PMH.Identify.deletedRecord"));
+ assertEquals("YYYY-MM-DDThh:mm:ssZ", responseXmlPath.getString("OAI-PMH.Identify.granularity"));
+ }
+
@Test
- public void testSetCreation() {
- setupUsers();
+ public void testOaiListMetadataFormats() {
+ // Run ListMeatadataFormats:
+ Response listFormatsResponse = UtilIT.getOaiListMetadataFormats();
+ assertEquals(OK.getStatusCode(), listFormatsResponse.getStatusCode());
+
+ // Validate the response:
+
+ XmlPath responseXmlPath = validateOaiVerbResponse(listFormatsResponse, "ListMetadataFormats");
+
+ // Check the payload of the response atgainst the list of metadata formats
+ // we are currently offering under OAI; will need to be explicitly
+ // modified if/when we add more harvestable formats.
+
+ List listFormats = responseXmlPath.getList("OAI-PMH.ListMetadataFormats.metadataFormat");
+
+ assertNotNull(listFormats);
+ assertEquals(5, listFormats.size());
+
+ // The metadata formats are reported in an unpredictable ordder. We
+ // want to sort the prefix names for comparison purposes, and for that
+ // they need to be saved in a modifiable list:
+ List metadataPrefixes = new ArrayList<>();
+
+ for (int i = 0; i < listFormats.size(); i++) {
+ metadataPrefixes.add(responseXmlPath.getString("OAI-PMH.ListMetadataFormats.metadataFormat["+i+"].metadataPrefix"));
+ }
+ Collections.sort(metadataPrefixes);
+
+ assertEquals("[Datacite, dataverse_json, oai_datacite, oai_dc, oai_ddi]", metadataPrefixes.toString());
+
+
+ }
+
+
+ @Test
+ public void testNativeSetAPI() {
String setName = UtilIT.getRandomString(6);
String def = "*";
-
- // make sure the set does not exist
- String u0 = String.format("/api/harvest/server/oaisets/%s", setName);
+
+ // This test focuses on the Create/List/Edit functionality of the
+ // Dataverse OAI Sets API (/api/harvest/server):
+
+ // API Test 1. Make sure the set does not exist yet
+ String setPath = String.format("/api/harvest/server/oaisets/%s", setName);
String createPath ="/api/harvest/server/oaisets/add";
- Response r0 = given()
- .get(u0);
- assertEquals(404, r0.getStatusCode());
+ Response getSetResponse = given()
+ .get(setPath);
+ assertEquals(404, getSetResponse.getStatusCode());
- // try to create set as normal user, should fail
- Response r1 = given()
+ // API Test 2. Try to create set as normal user, should fail
+ Response createSetResponse = given()
.header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey)
.body(jsonForTestSpec(setName, def))
.post(createPath);
- assertEquals(400, r1.getStatusCode());
+ assertEquals(400, createSetResponse.getStatusCode());
- // try to create set as admin user, should succeed
- Response r2 = given()
+ // API Test 3. Try to create set as admin user, should succeed
+ createSetResponse = given()
.header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey)
.body(jsonForTestSpec(setName, def))
.post(createPath);
- assertEquals(201, r2.getStatusCode());
+ assertEquals(201, createSetResponse.getStatusCode());
- Response getSet = given()
- .get(u0);
+ // API Test 4. Retrieve the set we've just created, validate the response
+ getSetResponse = given().get(setPath);
- logger.info("getSet.getStatusCode(): " + getSet.getStatusCode());
- logger.info("getSet printresponse: " + getSet.prettyPrint());
- assertEquals(200, getSet.getStatusCode());
+ System.out.println("getSetResponse.getStatusCode(): " + getSetResponse.getStatusCode());
+ System.out.println("getSetResponse, full: " + getSetResponse.prettyPrint());
+ assertEquals(200, getSetResponse.getStatusCode());
+
+ getSetResponse.then().assertThat()
+ .body("status", equalTo(AbstractApiBean.STATUS_OK))
+ .body("data.definition", equalTo("*"))
+ .body("data.description", equalTo(""))
+ .body("data.name", equalTo(setName));
+
+ // API Test 5. Retrieve all sets, check that our new set is listed
Response responseAll = given()
.get("/api/harvest/server/oaisets");
- logger.info("responseAll.getStatusCode(): " + responseAll.getStatusCode());
- logger.info("responseAll printresponse: " + responseAll.prettyPrint());
+ System.out.println("responseAll.getStatusCode(): " + responseAll.getStatusCode());
+ System.out.println("responseAll full: " + responseAll.prettyPrint());
assertEquals(200, responseAll.getStatusCode());
-
- // try to create set with same name as admin user, should fail
- Response r3 = given()
+ assertTrue(responseAll.body().jsonPath().getList("data.oaisets").size() > 0);
+ assertTrue(responseAll.body().jsonPath().getList("data.oaisets.name", String.class).contains(setName));
+
+ // API Test 6. Try to create a set with the same name, should fail
+ createSetResponse = given()
.header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey)
.body(jsonForTestSpec(setName, def))
.post(createPath);
- assertEquals(400, r3.getStatusCode());
+ assertEquals(400, createSetResponse.getStatusCode());
- // try to export set as admin user, should succeed (under admin API, not checking that normal user will fail)
+ // API Test 7. Try to export set as admin user, should succeed. Set export
+ // is under /api/admin, no need to try to access it as a non-admin user
Response r4 = UtilIT.exportOaiSet(setName);
assertEquals(200, r4.getStatusCode());
-
- // try to delete as normal user should fail
- Response r5 = given()
+
+ // API TEST 8. Try to delete the set as normal user, should fail
+ Response deleteResponse = given()
.header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey)
- .delete(u0);
- logger.info("r5.getStatusCode(): " + r5.getStatusCode());
- assertEquals(400, r5.getStatusCode());
+ .delete(setPath);
+ logger.info("deleteResponse.getStatusCode(): " + deleteResponse.getStatusCode());
+ assertEquals(400, deleteResponse.getStatusCode());
- // try to delete as admin user should work
- Response r6 = given()
+ // API TEST 9. Delete as admin user, should work
+ deleteResponse = given()
.header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey)
- .delete(u0);
- logger.info("r6.getStatusCode(): " + r6.getStatusCode());
- assertEquals(200, r6.getStatusCode());
+ .delete(setPath);
+ logger.info("deleteResponse.getStatusCode(): " + deleteResponse.getStatusCode());
+ assertEquals(200, deleteResponse.getStatusCode());
}
@Test
- public void testSetEdit() {
- setupUsers();
+ public void testSetEditAPIandOAIlistSets() {
+ // This test focuses on testing the Edit functionality of the Dataverse
+ // OAI Set API and the ListSets method of the Dataverse OAI server.
+
+ // Initial setup: crete a test set.
+ // Since the Create and List (POST and GET) functionality of the API
+ // is tested extensively in the previous test, we will not be paying
+ // as much attention to these methods, aside from confirming the
+ // expected HTTP result codes.
+
String setName = UtilIT.getRandomString(6);
- String def = "*";
+ String setDef = "*";
- // make sure the set does not exist
- String u0 = String.format("/api/harvest/server/oaisets/%s", setName);
+ // Make sure the set does not exist
+ String setPath = String.format("/api/harvest/server/oaisets/%s", setName);
String createPath ="/api/harvest/server/oaisets/add";
- Response r0 = given()
- .get(u0);
- assertEquals(404, r0.getStatusCode());
+ Response getSetResponse = given()
+ .get(setPath);
+ assertEquals(404, getSetResponse.getStatusCode());
- // try to create set as admin user, should succeed
- Response r1 = given()
+ // Create the set as admin user
+ Response createSetResponse = given()
.header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey)
- .body(jsonForTestSpec(setName, def))
+ .body(jsonForTestSpec(setName, setDef))
.post(createPath);
- assertEquals(201, r1.getStatusCode());
+ assertEquals(201, createSetResponse.getStatusCode());
+ // I. Test the Modify/Edit (POST method) functionality of the
+ // Dataverse OAI Sets API
+
+ String newDefinition = "title:New";
+ String newDescription = "updated";
- // try to edit as normal user should fail
- Response r2 = given()
+ // API Test 1. Try to modify the set as normal user, should fail
+ Response editSetResponse = given()
.header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey)
- .body(jsonForEditSpec(setName, def,""))
- .put(u0);
- logger.info("r2.getStatusCode(): " + r2.getStatusCode());
- assertEquals(400, r2.getStatusCode());
+ .body(jsonForEditSpec(setName, setDef, ""))
+ .put(setPath);
+ logger.info("non-admin user editSetResponse.getStatusCode(): " + editSetResponse.getStatusCode());
+ assertEquals(400, editSetResponse.getStatusCode());
- // try to edit as with blanks should fail
- Response r3 = given()
+ // API Test 2. Try to modify as admin, but with invalid (empty) values,
+ // should fail
+ editSetResponse = given()
.header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey)
.body(jsonForEditSpec(setName, "",""))
- .put(u0);
- logger.info("r3.getStatusCode(): " + r3.getStatusCode());
- assertEquals(400, r3.getStatusCode());
+ .put(setPath);
+ logger.info("invalid values editSetResponse.getStatusCode(): " + editSetResponse.getStatusCode());
+ assertEquals(400, editSetResponse.getStatusCode());
- // try to edit as with something should pass
- Response r4 = given()
+ // API Test 3. Try to modify as admin, with sensible values
+ editSetResponse = given()
.header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey)
- .body(jsonForEditSpec(setName, "newDef","newDesc"))
- .put(u0);
- logger.info("r4 Status code: " + r4.getStatusCode());
- logger.info("r4.prettyPrint(): " + r4.prettyPrint());
- assertEquals(OK.getStatusCode(), r4.getStatusCode());
-
- logger.info("u0: " + u0);
- // now delete it...
- Response r6 = given()
+ .body(jsonForEditSpec(setName, newDefinition, newDescription))
+ .put(setPath);
+ logger.info("admin user editSetResponse status code: " + editSetResponse.getStatusCode());
+ logger.info("admin user editSetResponse.prettyPrint(): " + editSetResponse.prettyPrint());
+ assertEquals(OK.getStatusCode(), editSetResponse.getStatusCode());
+
+ // API Test 4. List the set, confirm that the new values are shown
+ getSetResponse = given().get(setPath);
+
+ System.out.println("getSetResponse.getStatusCode(): " + getSetResponse.getStatusCode());
+ System.out.println("getSetResponse, full: " + getSetResponse.prettyPrint());
+ assertEquals(200, getSetResponse.getStatusCode());
+
+ getSetResponse.then().assertThat()
+ .body("status", equalTo(AbstractApiBean.STATUS_OK))
+ .body("data.definition", equalTo(newDefinition))
+ .body("data.description", equalTo(newDescription))
+ .body("data.name", equalTo(setName));
+
+ // II. Test the ListSets functionality of the OAI server
+
+ Response listSetsResponse = UtilIT.getOaiListSets();
+
+ // 1. Validate the service section of the OAI response:
+
+ XmlPath responseXmlPath = validateOaiVerbResponse(listSetsResponse, "ListSets");
+
+ // 2. Validate the payload of the response, by confirming that the set
+ // we created and modified, above, is being listed by the OAI server
+ // and its xml record is properly formatted
+
+ List listSets = responseXmlPath.getList("OAI-PMH.ListSets.set.list().findAll{it.setName=='"+setName+"'}", Node.class);
+
+ // 2a. Confirm that our set is listed:
+ assertNotNull("Unexpected response from ListSets", listSets);
+ assertTrue("Newly-created set isn't properly listed by the OAI server", listSets.size() == 1);
+ // 2b. Confirm that the set entry contains the updated description:
+ assertEquals("Incorrect description in the ListSets entry", newDescription, listSets.get(0).getPath("setDescription.metadata.element.field", String.class));
+
+ // ok, the xml record looks good!
+
+ // Cleanup. Delete the set with the DELETE API
+ Response deleteSetResponse = given()
.header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey)
- .delete(u0);
- logger.info("r6.getStatusCode(): " + r6.getStatusCode());
- assertEquals(200, r6.getStatusCode());
+ .delete(setPath);
+ assertEquals(200, deleteSetResponse.getStatusCode());
}
- // A more elaborate test - we'll create and publish a dataset, then create an
- // OAI set with that one dataset, and attempt to retrieve the OAI record
- // with GetRecord.
+ // A more elaborate test - we will create and export an
+ // OAI set with a single dataset, and attempt to retrieve
+ // it and validate the OAI server responses of the corresponding
+ // ListIdentifiers, ListRecords and GetRecord methods.
@Test
- public void testOaiFunctionality() throws InterruptedException {
-
- setupUsers();
-
- // create dataverse:
- Response createDataverseResponse = UtilIT.createRandomDataverse(adminUserAPIKey);
- createDataverseResponse.prettyPrint();
- String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
-
- // publish dataverse:
- Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, adminUserAPIKey);
- assertEquals(OK.getStatusCode(), publishDataverse.getStatusCode());
-
- // create dataset:
- Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, adminUserAPIKey);
- createDatasetResponse.prettyPrint();
- Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse);
-
- // retrieve the global id:
- String datasetPersistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse);
-
- // publish dataset:
- Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPersistentId, "major", adminUserAPIKey);
- assertEquals(200, publishDataset.getStatusCode());
-
- String identifier = datasetPersistentId.substring(datasetPersistentId.lastIndexOf('/') + 1);
-
- logger.info("identifier: " + identifier);
-
- // Let's try and create an OAI set with the dataset we have just
- // created and published:
- // - however, publish command is executed asynchronously, i.e. it may
- // still be running after we received the OK from the publish API.
- // The oaiExport step also requires the metadata exports to be done and this
- // takes longer than just publish/reindex.
- // So wait for all of this to finish.
- UtilIT.sleepForReexport(datasetPersistentId, adminUserAPIKey, 10);
+ public void testSingleRecordOaiSet() throws InterruptedException {
+ // Let's try and create an OAI set with the "single set dataset" that
+ // was created as part of the initial setup:
- String setName = identifier;
- String setQuery = "dsPersistentId:" + identifier;
+ String setName = singleSetDatasetIdentifier;
+ String setQuery = "dsPersistentId:" + singleSetDatasetIdentifier;
String apiPath = String.format("/api/harvest/server/oaisets/%s", setName);
String createPath ="/api/harvest/server/oaisets/add";
Response createSetResponse = given()
@@ -243,69 +407,470 @@ public void testOaiFunctionality() throws InterruptedException {
.post(createPath);
assertEquals(201, createSetResponse.getStatusCode());
- // TODO: a) look up the set via native harvest/server api;
- // b) look up the set via the OAI ListSets;
- // export set:
- // (this is asynchronous - so we should probably wait a little)
- Response exportSetResponse = UtilIT.exportOaiSet(setName);
- assertEquals(200, exportSetResponse.getStatusCode());
+ // The GET method of the oai set API, as well as the OAI ListSets
+ // method are tested extensively in another method in this class, so
+ // we'll skip looking too closely into those here.
+
+ // A quick test that the new set is listed under native API
Response getSet = given()
.get(apiPath);
-
- logger.info("getSet.getStatusCode(): " + getSet.getStatusCode());
- logger.fine("getSet printresponse: " + getSet.prettyPrint());
assertEquals(200, getSet.getStatusCode());
+
+ // Export the set.
+
+ Response exportSetResponse = UtilIT.exportOaiSet(setName);
+ assertEquals(200, exportSetResponse.getStatusCode());
+
+ // Strictly speaking, exporting an OAI set is an asynchronous operation.
+ // So the code below was written to expect to have to wait for up to 10
+ // additional seconds for it to complete. In retrospect, this is
+ // most likely unnecessary (because the only potentially expensive part
+ // of the process is the metadata export, and in this case that must have
+ // already happened - when the dataset was published (that operation
+ // now has its own wait mechanism). But I'll keep this extra code in
+ // place since it's not going to hurt. - L.A.
+
+ Thread.sleep(1000L); // initial sleep interval
int i = 0;
int maxWait=10;
do {
-
- // Run ListIdentifiers on this newly-created set:
+ // OAI Test 1. Run ListIdentifiers on this newly-created set:
Response listIdentifiersResponse = UtilIT.getOaiListIdentifiers(setName, "oai_dc");
- List ret = listIdentifiersResponse.getBody().xmlPath().getList("OAI-PMH.ListIdentifiers.header");
-
assertEquals(OK.getStatusCode(), listIdentifiersResponse.getStatusCode());
- assertNotNull(ret);
- logger.info("setName: " + setName);
- if (logger.isLoggable(Level.FINE)) {
- logger.info("listIdentifiersResponse.prettyPrint:..... ");
- listIdentifiersResponse.prettyPrint();
- }
- if (ret.size() != 1) {
+
+ // Validate the service section of the OAI response:
+ XmlPath responseXmlPath = validateOaiVerbResponse(listIdentifiersResponse, "ListIdentifiers");
+
+ List ret = responseXmlPath.getList("OAI-PMH.ListIdentifiers.header");
+
+ if (ret == null || ret.isEmpty()) {
+ // OK, we'll sleep for another second
i++;
} else {
- // There should be 1 and only 1 record in the response:
+ if (logger.isLoggable(Level.FINE)) {
+ logger.info("listIdentifiersResponse.prettyPrint: " + listIdentifiersResponse.prettyPrint());
+ }
+ // Validate the payload of the ListIdentifiers response:
+ // a) There should be 1 and only 1 item listed:
assertEquals(1, ret.size());
- // And the record should be the dataset we have just created:
- assertEquals(datasetPersistentId, listIdentifiersResponse.getBody().xmlPath()
+ // b) The one record in it should be the dataset we have just created:
+ assertEquals(singleSetDatasetPersistentId, responseXmlPath
.getString("OAI-PMH.ListIdentifiers.header.identifier"));
+ assertEquals(setName, responseXmlPath
+ .getString("OAI-PMH.ListIdentifiers.header.setSpec"));
+ assertNotNull(responseXmlPath.getString("OAI-PMH.ListIdentifiers.header.dateStamp"));
+ // TODO: validate the formatting of the date string here as well.
+
+ // ok, ListIdentifiers response looks valid.
break;
}
Thread.sleep(1000L);
- } while (i"));
- // And now run GetRecord on the OAI record for the dataset:
- Response getRecordResponse = UtilIT.getOaiRecord(datasetPersistentId, "oai_dc");
+ // OAI Test 4. run and validate GetRecord response
+
+ Response getRecordResponse = UtilIT.getOaiRecord(singleSetDatasetPersistentId, "oai_dc");
+ System.out.println("GetRecord response in its entirety: "+getRecordResponse.getBody().prettyPrint());
+
+ // Validate the service section of the OAI response:
+ responseXmlPath = validateOaiVerbResponse(getRecordResponse, "GetRecord");
+
+ // Validate the payload of the response:
+
+ // Note that for a set with a single record the output of ListRecrods is
+ // essentially identical to that of GetRecord!
+ // (we'll test a multi-record set in a different method)
+ // a) header section:
+ assertEquals(singleSetDatasetPersistentId, responseXmlPath.getString("OAI-PMH.GetRecord.record.header.identifier"));
+ assertEquals(setName, responseXmlPath
+ .getString("OAI-PMH.GetRecord.record.header.setSpec"));
+ assertNotNull(responseXmlPath.getString("OAI-PMH.GetRecord.record.header.dateStamp"));
+ // b) metadata section:
+ assertEquals(persistentIdUrl, responseXmlPath.getString("OAI-PMH.GetRecord.record.metadata.dc.identifier"));
+ assertEquals("Darwin's Finches", responseXmlPath.getString("OAI-PMH.GetRecord.record.metadata.dc.title"));
+ assertEquals("Finch, Fiona", responseXmlPath.getString("OAI-PMH.GetRecord.record.metadata.dc.creator"));
+ assertEquals("Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds.",
+ responseXmlPath.getString("OAI-PMH.GetRecord.record.metadata.dc.description"));
+ assertEquals("Medicine, Health and Life Sciences", responseXmlPath.getString("OAI-PMH.GetRecord.record.metadata.dc.subject"));
+
+ // ok, looks legit!
+
+ }
+
+ // This test will attempt to create a set with multiple records (enough
+ // to trigger a paged respons) and test the resumption token functionality).
+ // Note that this test requires the OAI service to be configured with some
+ // non-default settings (the paging limits for ListIdentifiers and ListRecords
+ // must be set to 2, in order to be able to trigger this paging behavior without
+ // having to create and export too many datasets).
+ // So you will need to do this:
+ // asadmin create-jvm-options "-Ddataverse.oai.server.maxidentifiers=2"
+ // asadmin create-jvm-options "-Ddataverse.oai.server.maxrecords=2"
+
+
+ @Test
+ public void testMultiRecordOaiSet() throws InterruptedException {
+ // Setup: Let's create a control OAI set with the 5 datasets created
+ // in the class init:
+
+ String setName = UtilIT.getRandomString(6);
+ String setQuery = "(dsPersistentId:" + singleSetDatasetIdentifier;
+ for (String persistentId : extraDatasetsIdentifiers) {
+ setQuery = setQuery.concat(" OR dsPersistentId:" + persistentId);
+ }
+ setQuery = setQuery.concat(")");
+
+ String createPath = "/api/harvest/server/oaisets/add";
+
+ Response createSetResponse = given()
+ .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey)
+ .body(jsonForTestSpec(setName, setQuery))
+ .post(createPath);
+ assertEquals(201, createSetResponse.getStatusCode());
+
+ // Dataverse OAI Sets API is tested extensively in other methods here,
+ // so no need to test in any more details than confirming the OK result
+ // above
+ Response exportSetResponse = UtilIT.exportOaiSet(setName);
+ assertEquals(200, exportSetResponse.getStatusCode());
+ Thread.sleep(1000L);
+
+ // OAI Test 1. Run ListIdentifiers on the set we've just created:
+ Response listIdentifiersResponse = UtilIT.getOaiListIdentifiers(setName, "oai_dc");
+ assertEquals(OK.getStatusCode(), listIdentifiersResponse.getStatusCode());
+
+ // Validate the service section of the OAI response:
+ XmlPath responseXmlPath = validateOaiVerbResponse(listIdentifiersResponse, "ListIdentifiers");
- assertEquals(datasetPersistentId, getRecordResponse.getBody().xmlPath().getString("OAI-PMH.GetRecord.record.header.identifier"));
+ List ret = responseXmlPath.getList("OAI-PMH.ListIdentifiers.header.identifier");
+ assertNotNull(ret);
- // TODO:
- // check the actual metadata payload of the OAI record more carefully?
+ if (logger.isLoggable(Level.FINE)) {
+ logger.info("listIdentifiersResponse.prettyPrint: "+listIdentifiersResponse.prettyPrint());
+ }
+
+ // Validate the payload of the ListIdentifiers response:
+ // 1a) There should be 2 items listed:
+ assertEquals("Wrong number of items on the first ListIdentifiers page",
+ 2, ret.size());
+
+ // 1b) The response contains a resumptionToken for the next page of items:
+ String resumptionToken = responseXmlPath.getString("OAI-PMH.ListIdentifiers.resumptionToken");
+ assertNotNull("No resumption token in the ListIdentifiers response (has the jvm option dataverse.oai.server.maxidentifiers been configured?)", resumptionToken);
+
+ // 1c) The total number of items in the set (5) is listed correctly:
+ assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@completeListSize"));
+
+ // 1d) ... and the offset (cursor) is at the right position (0):
+ assertEquals(0, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@cursor"));
+
+ // The formatting of individual item records in the ListIdentifiers response
+ // is tested extensively in the previous test method, so we are not
+ // looking at them in such detail here; but we should record the
+ // identifiers listed, so that we can confirm that all the set is
+ // served as expected.
+
+ Set persistentIdsInListIdentifiers = new HashSet<>();
+
+ for (String persistentId : ret) {
+ persistentIdsInListIdentifiers.add(persistentId.substring(persistentId.lastIndexOf('/') + 1));
+ }
+
+ // ok, let's move on to the next ListIdentifiers page:
+ // (we repeat the exact same checks as the above; minus the different
+ // expected offset)
+
+ // OAI Test 2. Run ListIdentifiers with the resumptionToken obtained
+ // in the previous step:
+
+ listIdentifiersResponse = UtilIT.getOaiListIdentifiersWithResumptionToken(resumptionToken);
+ assertEquals(OK.getStatusCode(), listIdentifiersResponse.getStatusCode());
+
+ // Validate the service section of the OAI response:
+ responseXmlPath = validateOaiVerbResponse(listIdentifiersResponse, "ListIdentifiers");
+
+ ret = responseXmlPath.getList("OAI-PMH.ListIdentifiers.header.identifier");
+ assertNotNull(ret);
+
+ if (logger.isLoggable(Level.FINE)) {
+ logger.info("listIdentifiersResponse.prettyPrint: "+listIdentifiersResponse.prettyPrint());
+ }
+
+ // Validate the payload of the ListIdentifiers response:
+ // 2a) There should still be 2 items listed:
+ assertEquals("Wrong number of items on the second ListIdentifiers page",
+ 2, ret.size());
+
+ // 2b) The response should contain a resumptionToken for the next page of items:
+ resumptionToken = responseXmlPath.getString("OAI-PMH.ListIdentifiers.resumptionToken");
+ assertNotNull("No resumption token in the ListIdentifiers response", resumptionToken);
+
+ // 2c) The total number of items in the set (5) is listed correctly:
+ assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@completeListSize"));
+
+ // 2d) ... and the offset (cursor) is at the right position (2):
+ assertEquals(2, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@cursor"));
+
+ // Record the identifiers listed on this results page:
+
+ for (String persistentId : ret) {
+ persistentIdsInListIdentifiers.add(persistentId.substring(persistentId.lastIndexOf('/') + 1));
+ }
+
+ // And now the next and the final ListIdentifiers page.
+ // This time around we should get an *empty* resumptionToken (indicating
+ // that there are no more results):
+
+ // OAI Test 3. Run ListIdentifiers with the final resumptionToken
+
+ listIdentifiersResponse = UtilIT.getOaiListIdentifiersWithResumptionToken(resumptionToken);
+ assertEquals(OK.getStatusCode(), listIdentifiersResponse.getStatusCode());
+
+ // Validate the service section of the OAI response:
+ responseXmlPath = validateOaiVerbResponse(listIdentifiersResponse, "ListIdentifiers");
+
+ ret = responseXmlPath.getList("OAI-PMH.ListIdentifiers.header.identifier");
+ assertNotNull(ret);
+
+ if (logger.isLoggable(Level.FINE)) {
+ logger.info("listIdentifiersResponse.prettyPrint: "+listIdentifiersResponse.prettyPrint());
+ }
+
+ // Validate the payload of the ListIdentifiers response:
+ // 3a) There should be only 1 item listed:
+ assertEquals("Wrong number of items on the final ListIdentifiers page",
+ 1, ret.size());
+
+ // 3b) The response contains a resumptionToken for the next page of items:
+ resumptionToken = responseXmlPath.getString("OAI-PMH.ListIdentifiers.resumptionToken");
+ assertNotNull("No resumption token in the final ListIdentifiers response", resumptionToken);
+ assertTrue("Non-empty resumption token in the final ListIdentifiers response", "".equals(resumptionToken));
+
+ // 3c) The total number of items in the set (5) is still listed correctly:
+ assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@completeListSize"));
+
+ // 3d) ... and the offset (cursor) is at the right position (4):
+ assertEquals(4, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@cursor"));
+
+ // Record the last identifier listed on this final page:
+ persistentIdsInListIdentifiers.add(ret.get(0).substring(ret.get(0).lastIndexOf('/') + 1));
+
+ // Finally, let's confirm that the expected 5 datasets have been listed
+ // as part of this Set:
+
+ boolean allDatasetsListed = true;
+
+ allDatasetsListed = persistentIdsInListIdentifiers.contains(singleSetDatasetIdentifier);
+ for (String persistentId : extraDatasetsIdentifiers) {
+ allDatasetsListed = allDatasetsListed && persistentIdsInListIdentifiers.contains(persistentId);
+ }
+
+ assertTrue("Control datasets not properly listed in the paged ListIdentifiers response",
+ allDatasetsListed);
+
+ // OK, it is safe to assume ListIdentifiers works as it should in page mode.
+
+ // We will now repeat the exact same tests for ListRecords (again, no
+ // need to pay close attention to the formatting of the individual records,
+ // since that's tested in the previous test method, since our focus is
+ // testing the paging/resumptionToken functionality)
+
+ // OAI Test 4. Run ListRecords on the set we've just created:
+ Response listRecordsResponse = UtilIT.getOaiListRecords(setName, "oai_dc");
+ assertEquals(OK.getStatusCode(), listRecordsResponse.getStatusCode());
+
+ // Validate the service section of the OAI response:
+ responseXmlPath = validateOaiVerbResponse(listRecordsResponse, "ListRecords");
+
+ ret = responseXmlPath.getList("OAI-PMH.ListRecords.record.header.identifier");
+ assertNotNull(ret);
+
+ if (logger.isLoggable(Level.FINE)) {
+ logger.info("listRecordsResponse.prettyPrint: "+listRecordsResponse.prettyPrint());
+ }
+
+ // Validate the payload of the ListRecords response:
+ // 4a) There should be 2 items listed:
+ assertEquals("Wrong number of items on the first ListRecords page",
+ 2, ret.size());
+
+ // 4b) The response contains a resumptionToken for the next page of items:
+ resumptionToken = responseXmlPath.getString("OAI-PMH.ListRecords.resumptionToken");
+ assertNotNull("No resumption token in the ListRecords response (has the jvm option dataverse.oai.server.maxrecords been configured?)", resumptionToken);
+
+ // 4c) The total number of items in the set (5) is listed correctly:
+ assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@completeListSize"));
+
+ // 4d) ... and the offset (cursor) is at the right position (0):
+ assertEquals(0, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@cursor"));
+
+ Set persistentIdsInListRecords = new HashSet<>();
+
+ for (String persistentId : ret) {
+ persistentIdsInListRecords.add(persistentId.substring(persistentId.lastIndexOf('/') + 1));
+ }
+
+ // ok, let's move on to the next ListRecords page:
+ // (we repeat the exact same checks as the above; minus the different
+ // expected offset)
+
+ // OAI Test 5. Run ListRecords with the resumptionToken obtained
+ // in the previous step:
+
+ listRecordsResponse = UtilIT.getOaiListRecordsWithResumptionToken(resumptionToken);
+ assertEquals(OK.getStatusCode(), listRecordsResponse.getStatusCode());
+
+ // Validate the service section of the OAI response:
+ responseXmlPath = validateOaiVerbResponse(listRecordsResponse, "ListRecords");
+
+ ret = responseXmlPath.getList("OAI-PMH.ListRecords.record.header.identifier");
+ assertNotNull(ret);
+
+ if (logger.isLoggable(Level.FINE)) {
+ logger.info("listRecordsResponse.prettyPrint: "+listRecordsResponse.prettyPrint());
+ }
+
+ // Validate the payload of the ListRecords response:
+ // 4a) There should still be 2 items listed:
+ assertEquals("Wrong number of items on the second ListRecords page",
+ 2, ret.size());
+
+ // 4b) The response should contain a resumptionToken for the next page of items:
+ resumptionToken = responseXmlPath.getString("OAI-PMH.ListRecords.resumptionToken");
+ assertNotNull("No resumption token in the ListRecords response", resumptionToken);
+
+ // 4c) The total number of items in the set (5) is listed correctly:
+ assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@completeListSize"));
+
+ // 4d) ... and the offset (cursor) is at the right position (2):
+ assertEquals(2, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@cursor"));
+
+ // Record the identifiers listed on this results page:
+
+ for (String persistentId : ret) {
+ persistentIdsInListRecords.add(persistentId.substring(persistentId.lastIndexOf('/') + 1));
+ }
+
+ // And now the next and the final ListRecords page.
+ // This time around we should get an *empty* resumptionToken (indicating
+ // that there are no more results):
+
+ // OAI Test 6. Run ListRecords with the final resumptionToken
+
+ listRecordsResponse = UtilIT.getOaiListRecordsWithResumptionToken(resumptionToken);
+ assertEquals(OK.getStatusCode(), listRecordsResponse.getStatusCode());
+
+ // Validate the service section of the OAI response:
+ responseXmlPath = validateOaiVerbResponse(listRecordsResponse, "ListRecords");
+
+ ret = responseXmlPath.getList("OAI-PMH.ListRecords.record.header.identifier");
+ assertNotNull(ret);
+
+ if (logger.isLoggable(Level.FINE)) {
+ logger.info("listRecordsResponse.prettyPrint: "+listRecordsResponse.prettyPrint());
+ }
+
+ // Validate the payload of the ListRecords response:
+ // 6a) There should be only 1 item listed:
+ assertEquals("Wrong number of items on the final ListRecords page",
+ 1, ret.size());
+
+ // 6b) The response contains a resumptionToken for the next page of items:
+ resumptionToken = responseXmlPath.getString("OAI-PMH.ListRecords.resumptionToken");
+ assertNotNull("No resumption token in the final ListRecords response", resumptionToken);
+ assertTrue("Non-empty resumption token in the final ListRecords response", "".equals(resumptionToken));
+
+ // 6c) The total number of items in the set (5) is still listed correctly:
+ assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@completeListSize"));
+
+ // 6d) ... and the offset (cursor) is at the right position (4):
+ assertEquals(4, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@cursor"));
+
+ // Record the last identifier listed on this final page:
+ persistentIdsInListRecords.add(ret.get(0).substring(ret.get(0).lastIndexOf('/') + 1));
+
+ // Finally, let's confirm that the expected 5 datasets have been listed
+ // as part of this Set:
+
+ allDatasetsListed = true;
+
+ allDatasetsListed = persistentIdsInListRecords.contains(singleSetDatasetIdentifier);
+ for (String persistentId : extraDatasetsIdentifiers) {
+ allDatasetsListed = allDatasetsListed && persistentIdsInListRecords.contains(persistentId);
+ }
+
+ assertTrue("Control datasets not properly listed in the paged ListRecords response",
+ allDatasetsListed);
+
+ // OK, it is safe to assume ListRecords works as it should in page mode
+ // as well.
+
+ // And finally, let's delete the set
+ String setPath = String.format("/api/harvest/server/oaisets/%s", setName);
+ Response deleteResponse = given()
+ .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey)
+ .delete(setPath);
+ logger.info("deleteResponse.getStatusCode(): " + deleteResponse.getStatusCode());
+ assertEquals("Failed to delete the control multi-record set", 200, deleteResponse.getStatusCode());
}
+
+ // TODO:
+ // What else can we test?
+ // Some ideas:
+ // - Test handling of deleted dataset records
+ // - Test "from" and "until" time parameters
+ // - Validate full verb response records against XML schema
+ // (for each supported metadata format, possibly?)
}
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 54a217be527..12ccaf2caff 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -2625,13 +2625,39 @@ static Response exportOaiSet(String setName) {
return given().put(apiPath);
}
+ static Response getOaiIdentify() {
+ String oaiVerbPath = "/oai?verb=Identify";
+ return given().get(oaiVerbPath);
+ }
+
+ static Response getOaiListMetadataFormats() {
+ String oaiVerbPath = "/oai?verb=ListMetadataFormats";
+ return given().get(oaiVerbPath);
+ }
+
+ static Response getOaiListSets() {
+ String oaiVerbPath = "/oai?verb=ListSets";
+ return given().get(oaiVerbPath);
+ }
+
static Response getOaiRecord(String datasetPersistentId, String metadataFormat) {
String apiPath = String.format("/oai?verb=GetRecord&identifier=%s&metadataPrefix=%s", datasetPersistentId, metadataFormat);
return given().get(apiPath);
}
static Response getOaiListIdentifiers(String setName, String metadataFormat) {
- String apiPath = String.format("/oai?verb=ListIdentifiers&set=%s&metadataPrefix=%s", setName, metadataFormat);
+
+ String apiPath;
+ if (StringUtil.nonEmpty(setName)) {
+ apiPath = String.format("/oai?verb=ListIdentifiers&set=%s&metadataPrefix=%s", setName, metadataFormat);
+ } else {
+ apiPath = String.format("/oai?verb=ListIdentifiers&metadataPrefix=%s", metadataFormat);
+ }
+ return given().get(apiPath);
+ }
+
+ static Response getOaiListIdentifiersWithResumptionToken(String resumptionToken) {
+ String apiPath = String.format("/oai?verb=ListIdentifiers&resumptionToken=%s", resumptionToken);
return given().get(apiPath);
}
@@ -2639,6 +2665,11 @@ static Response getOaiListRecords(String setName, String metadataFormat) {
String apiPath = String.format("/oai?verb=ListRecords&set=%s&metadataPrefix=%s", setName, metadataFormat);
return given().get(apiPath);
}
+
+ static Response getOaiListRecordsWithResumptionToken(String resumptionToken) {
+ String apiPath = String.format("/oai?verb=ListRecords&resumptionToken=%s", resumptionToken);
+ return given().get(apiPath);
+ }
static Response changeAuthenticatedUserIdentifier(String oldIdentifier, String newIdentifier, String apiToken) {
Response response;