From 175ab7fd29a7a51a95811963113d332c8f2621c3 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 2 Dec 2022 13:06:08 -0500 Subject: [PATCH 01/11] rc2 version of the gdcc-xoai lib. should've been done a long time ago :( #8843 --- modules/dataverse-parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index e36a78b11be..bf37299f2df 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -163,7 +163,7 @@ 4.4.14 - 5.0.0-RC1 + 5.0.0-RC2 1.15.0 From 3d1e98c5a9f5f755d8d78b6151b659fe2377f3ed Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 2 Dec 2022 13:27:40 -0500 Subject: [PATCH 02/11] this method was renamed in RC2 (#8843) --- .../harvest/server/xoai/DataverseXoaiItemRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java index faf3cf9ddc4..147d42648fa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java @@ -49,7 +49,7 @@ public DataverseXoaiItemRepository (OAIRecordServiceBean recordService, DatasetS } @Override - public ItemIdentifier getItem(String identifier) throws IdDoesNotExistException { + public ItemIdentifier getItemIdentifier(String identifier) throws IdDoesNotExistException { // This method is called when ListMetadataFormats request specifies // the identifier, requesting the formats available for this specific record. // In our case, under the current implementation, we need to simply look From aeffa3b6fc13a029b70630d856b5f0373a333903 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 5 Dec 2022 20:41:24 -0500 Subject: [PATCH 03/11] a few extra oai tests (#8843) --- .../iq/dataverse/api/HarvestingServerIT.java | 222 +++++++++++++----- .../edu/harvard/iq/dataverse/api/UtilIT.java | 10 + 2 files changed, 176 insertions(+), 56 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index fdd034ab12e..5355b57490d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -10,7 +10,12 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import com.jayway.restassured.response.Response; import com.jayway.restassured.path.json.JsonPath; +import com.jayway.restassured.path.xml.XmlPath; +import com.jayway.restassured.path.xml.element.Node; import static edu.harvard.iq.dataverse.api.UtilIT.API_TOKEN_HTTP_HEADER; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import javax.json.Json; import javax.json.JsonArray; import static javax.ws.rs.core.Response.Status.FORBIDDEN; @@ -24,18 +29,32 @@ import static org.junit.Assert.assertTrue; /** - * extremely minimal API tests for creating OAI sets. + * Tests for the Harvesting Server functionality + * Note that these test BOTH the proprietary Dataverse rest APIs for creating + * and managing sets, AND the OAI-PMH functionality itself. */ public class HarvestingServerIT { private static final Logger logger = Logger.getLogger(HarvestingServerIT.class.getCanonicalName()); + private static String normalUserAPIKey; + private static String adminUserAPIKey; + private static String singleSetDatasetIdentifier; + private static String singleSetDatasetPersistentId; + @BeforeClass public static void setUpClass() { RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); // enable harvesting server // Gave some thought to storing the original response, and resetting afterwards - but that appears to be more complexity than it's worth Response enableHarvestingServerResponse = UtilIT.setSetting(SettingsServiceBean.Key.OAIServerEnabled,"true"); + + // Create users: + setupUsers(); + + // Create and publish some datasets: + setupDatasets(); + } @AfterClass @@ -44,7 +63,7 @@ public static void afterClass() { Response enableHarvestingServerResponse = UtilIT.setSetting(SettingsServiceBean.Key.OAIServerEnabled,"false"); } - private void setupUsers() { + private static void setupUsers() { Response cu0 = UtilIT.createRandomUser(); normalUserAPIKey = UtilIT.getApiTokenFromResponse(cu0); Response cu1 = UtilIT.createRandomUser(); @@ -52,6 +71,40 @@ private void setupUsers() { Response u1a = UtilIT.makeSuperUser(un1); adminUserAPIKey = UtilIT.getApiTokenFromResponse(cu1); } + + private static void setupDatasets() { + // create dataverse: + Response createDataverseResponse = UtilIT.createRandomDataverse(adminUserAPIKey); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + // publish dataverse: + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, adminUserAPIKey); + assertEquals(OK.getStatusCode(), publishDataverse.getStatusCode()); + + // create dataset: + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, adminUserAPIKey); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + + // retrieve the global id: + singleSetDatasetPersistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse); + + // publish dataset: + Response publishDataset = UtilIT.publishDatasetViaNativeApi(singleSetDatasetPersistentId, "major", adminUserAPIKey); + assertEquals(200, publishDataset.getStatusCode()); + + singleSetDatasetIdentifier = singleSetDatasetPersistentId.substring(singleSetDatasetPersistentId.lastIndexOf('/') + 1); + + logger.info("identifier: " + singleSetDatasetIdentifier); + + // Publish command is executed asynchronously, i.e. it may + // still be running after we received the OK from the publish API. + // The oaiExport step also requires the metadata exports to be done and this + // takes longer than just publish/reindex. + // So wait for all of this to finish. + UtilIT.sleepForReexport(singleSetDatasetPersistentId, adminUserAPIKey, 10); + } private String jsonForTestSpec(String name, String def) { String r = String.format("{\"name\":\"%s\",\"definition\":\"%s\"}", name, def);//description is optional @@ -63,20 +116,84 @@ private String jsonForEditSpec(String name, String def, String desc) { return r; } - private String normalUserAPIKey; - private String adminUserAPIKey; + private XmlPath validateOaiVerbResponse(Response oaiResponse, String verb) { + // confirm that the response is in fact XML: + XmlPath responseXmlPath = oaiResponse.getBody().xmlPath(); + assertNotNull(responseXmlPath); + + String dateString = responseXmlPath.getString("OAI-PMH.responseDate"); + assertNotNull(dateString); // TODO: validate that it's well-formatted! + logger.info("date string from the OAI output:"+dateString); + assertEquals("http://localhost:8080/oai", responseXmlPath.getString("OAI-PMH.request")); + assertEquals(verb, responseXmlPath.getString("OAI-PMH.request.@verb")); + return responseXmlPath; + } + + @Test + public void testOaiIdentify() { + // Run Identify: + Response identifyResponse = UtilIT.getOaiIdentify(); + assertEquals(OK.getStatusCode(), identifyResponse.getStatusCode()); + //logger.info("Identify response: "+identifyResponse.prettyPrint()); + + // Validate the response: + + XmlPath responseXmlPath = validateOaiVerbResponse(identifyResponse, "Identify"); + assertEquals("http://localhost:8080/oai", responseXmlPath.getString("OAI-PMH.Identify.baseURL")); + // Confirm that the server is reporting the correct parameters that + // our server implementation should be using: + assertEquals("2.0", responseXmlPath.getString("OAI-PMH.Identify.protocolVersion")); + assertEquals("transient", responseXmlPath.getString("OAI-PMH.Identify.deletedRecord")); + assertEquals("YYYY-MM-DDThh:mm:ssZ", responseXmlPath.getString("OAI-PMH.Identify.granularity")); + } + + @Test + public void testOaiListMetadataFormats() { + // Run ListMeatadataFormats: + Response listFormatsResponse = UtilIT.getOaiListMetadataFormats(); + assertEquals(OK.getStatusCode(), listFormatsResponse.getStatusCode()); + //logger.info("ListMetadataFormats response: "+listFormatsResponse.prettyPrint()); + + // Validate the response: + + XmlPath responseXmlPath = validateOaiVerbResponse(listFormatsResponse, "ListMetadataFormats"); + + // Check the payload of the response atgainst the list of metadata formats + // we are currently offering under OAI; will need to be explicitly + // modified if/when we add more harvestable formats. + + List listFormats = responseXmlPath.getList("OAI-PMH.ListMetadataFormats.metadataFormat"); + + assertNotNull(listFormats); + assertEquals(5, listFormats.size()); + + // The metadata formats are reported in an unpredictable ordder. We + // want to sort the prefix names for comparison purposes, and for that + // they need to be saved in a modifiable list: + List metadataPrefixes = new ArrayList<>(); + + for (int i = 0; i < listFormats.size(); i++) { + metadataPrefixes.add(responseXmlPath.getString("OAI-PMH.ListMetadataFormats.metadataFormat["+i+"].metadataPrefix")); + } + Collections.sort(metadataPrefixes); + + assertEquals("[Datacite, dataverse_json, oai_datacite, oai_dc, oai_ddi]", metadataPrefixes.toString()); + + } + + @Test - public void testSetCreation() { - setupUsers(); + public void testSetCreateAPIandOAIlistIdentifiers() { + // Create the set with Dataverse /api/harvest/server API: String setName = UtilIT.getRandomString(6); String def = "*"; // make sure the set does not exist - String u0 = String.format("/api/harvest/server/oaisets/%s", setName); + String setPath = String.format("/api/harvest/server/oaisets/%s", setName); String createPath ="/api/harvest/server/oaisets/add"; Response r0 = given() - .get(u0); + .get(setPath); assertEquals(404, r0.getStatusCode()); // try to create set as normal user, should fail @@ -94,7 +211,7 @@ public void testSetCreation() { assertEquals(201, r2.getStatusCode()); Response getSet = given() - .get(u0); + .get(setPath); logger.info("getSet.getStatusCode(): " + getSet.getStatusCode()); logger.info("getSet printresponse: " + getSet.prettyPrint()); @@ -118,17 +235,19 @@ public void testSetCreation() { Response r4 = UtilIT.exportOaiSet(setName); assertEquals(200, r4.getStatusCode()); - // try to delete as normal user should fail + + + // try to delete as normal user, should fail Response r5 = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey) - .delete(u0); + .delete(setPath); logger.info("r5.getStatusCode(): " + r5.getStatusCode()); assertEquals(400, r5.getStatusCode()); - // try to delete as admin user should work + // try to delete as admin user, should work Response r6 = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) - .delete(u0); + .delete(setPath); logger.info("r6.getStatusCode(): " + r6.getStatusCode()); assertEquals(200, r6.getStatusCode()); @@ -136,7 +255,7 @@ public void testSetCreation() { @Test public void testSetEdit() { - setupUsers(); + //setupUsers(); String setName = UtilIT.getRandomString(6); String def = "*"; @@ -195,46 +314,17 @@ public void testSetEdit() { // OAI set with that one dataset, and attempt to retrieve the OAI record // with GetRecord. @Test - public void testOaiFunctionality() throws InterruptedException { + public void testSingleRecordOaiSet() throws InterruptedException { - setupUsers(); - - // create dataverse: - Response createDataverseResponse = UtilIT.createRandomDataverse(adminUserAPIKey); - createDataverseResponse.prettyPrint(); - String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + //setupUsers(); - // publish dataverse: - Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, adminUserAPIKey); - assertEquals(OK.getStatusCode(), publishDataverse.getStatusCode()); - - // create dataset: - Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, adminUserAPIKey); - createDatasetResponse.prettyPrint(); - Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); - - // retrieve the global id: - String datasetPersistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse); - - // publish dataset: - Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPersistentId, "major", adminUserAPIKey); - assertEquals(200, publishDataset.getStatusCode()); - - String identifier = datasetPersistentId.substring(datasetPersistentId.lastIndexOf('/') + 1); - - logger.info("identifier: " + identifier); + - // Let's try and create an OAI set with the dataset we have just - // created and published: - // - however, publish command is executed asynchronously, i.e. it may - // still be running after we received the OK from the publish API. - // The oaiExport step also requires the metadata exports to be done and this - // takes longer than just publish/reindex. - // So wait for all of this to finish. - UtilIT.sleepForReexport(datasetPersistentId, adminUserAPIKey, 10); + // Let's try and create an OAI set with the "single set dataset" that + // was created as part of the initial setup: - String setName = identifier; - String setQuery = "dsPersistentId:" + identifier; + String setName = singleSetDatasetIdentifier; + String setQuery = "dsPersistentId:" + singleSetDatasetIdentifier; String apiPath = String.format("/api/harvest/server/oaisets/%s", setName); String createPath ="/api/harvest/server/oaisets/add"; Response createSetResponse = given() @@ -277,12 +367,18 @@ public void testOaiFunctionality() throws InterruptedException { // There should be 1 and only 1 record in the response: assertEquals(1, ret.size()); // And the record should be the dataset we have just created: - assertEquals(datasetPersistentId, listIdentifiersResponse.getBody().xmlPath() + assertEquals(singleSetDatasetPersistentId, listIdentifiersResponse.getBody().xmlPath() .getString("OAI-PMH.ListIdentifiers.header.identifier")); break; } Thread.sleep(1000L); - } while (i")); // And now run GetRecord on the OAI record for the dataset: - Response getRecordResponse = UtilIT.getOaiRecord(datasetPersistentId, "oai_dc"); - - assertEquals(datasetPersistentId, getRecordResponse.getBody().xmlPath().getString("OAI-PMH.GetRecord.record.header.identifier")); + Response getRecordResponse = UtilIT.getOaiRecord(singleSetDatasetPersistentId, "oai_dc"); + + System.out.println("GetRecord response in its entirety: "+getRecordResponse.getBody().prettyPrint()); + System.out.println("one more time:"); + getRecordResponse.prettyPrint(); + + assertEquals(singleSetDatasetPersistentId, getRecordResponse.getBody().xmlPath().getString("OAI-PMH.GetRecord.record.header.identifier")); // TODO: // check the actual metadata payload of the OAI record more carefully? } + + // This test will attempt to create a set with multiple records (enough + // to trigger a paged response with a continuation token) and test its + // performance. + + + @Test + public void testMultiRecordOaiSet() throws InterruptedException { + + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 550d4ed1264..9fa47db167b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2620,6 +2620,16 @@ static Response exportOaiSet(String setName) { return given().put(apiPath); } + static Response getOaiIdentify() { + String oaiVerbPath = "/oai?verb=Identify"; + return given().get(oaiVerbPath); + } + + static Response getOaiListMetadataFormats() { + String oaiVerbPath = "/oai?verb=ListMetadataFormats"; + return given().get(oaiVerbPath); + } + static Response getOaiRecord(String datasetPersistentId, String metadataFormat) { String apiPath = String.format("/oai?verb=GetRecord&identifier=%s&metadataPrefix=%s", datasetPersistentId, metadataFormat); return given().get(apiPath); From c4f07f91446eedeee611a75537b3b90872817d0b Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 9 Dec 2022 17:57:29 -0500 Subject: [PATCH 04/11] more tests for the OAI server functionality (#8843) --- .../iq/dataverse/api/HarvestingServerIT.java | 349 ++++++++++++------ .../edu/harvard/iq/dataverse/api/UtilIT.java | 5 + 2 files changed, 243 insertions(+), 111 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index 5355b57490d..d25ffd225d9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -9,24 +9,18 @@ import org.junit.Test; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import com.jayway.restassured.response.Response; -import com.jayway.restassured.path.json.JsonPath; import com.jayway.restassured.path.xml.XmlPath; import com.jayway.restassured.path.xml.element.Node; -import static edu.harvard.iq.dataverse.api.UtilIT.API_TOKEN_HTTP_HEADER; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import javax.json.Json; -import javax.json.JsonArray; -import static javax.ws.rs.core.Response.Status.FORBIDDEN; import static javax.ws.rs.core.Response.Status.OK; import static org.hamcrest.CoreMatchers.equalTo; -import org.junit.Ignore; import java.util.List; -import static junit.framework.Assert.assertEquals; +//import static junit.framework.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; /** * Tests for the Harvesting Server functionality @@ -184,142 +178,204 @@ public void testOaiListMetadataFormats() { @Test - public void testSetCreateAPIandOAIlistIdentifiers() { - // Create the set with Dataverse /api/harvest/server API: + public void testNativeSetAPI() { String setName = UtilIT.getRandomString(6); String def = "*"; - - // make sure the set does not exist + + // This test focuses on the Create/List/Edit functionality of the + // Dataverse OAI Sets API (/api/harvest/server): + + // API Test 1. Make sure the set does not exist yet String setPath = String.format("/api/harvest/server/oaisets/%s", setName); String createPath ="/api/harvest/server/oaisets/add"; - Response r0 = given() + Response getSetResponse = given() .get(setPath); - assertEquals(404, r0.getStatusCode()); + assertEquals(404, getSetResponse.getStatusCode()); - // try to create set as normal user, should fail - Response r1 = given() + // API Test 2. Try to create set as normal user, should fail + Response createSetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey) .body(jsonForTestSpec(setName, def)) .post(createPath); - assertEquals(400, r1.getStatusCode()); + assertEquals(400, createSetResponse.getStatusCode()); - // try to create set as admin user, should succeed - Response r2 = given() + // API Test 3. Try to create set as admin user, should succeed + createSetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) .body(jsonForTestSpec(setName, def)) .post(createPath); - assertEquals(201, r2.getStatusCode()); + assertEquals(201, createSetResponse.getStatusCode()); - Response getSet = given() - .get(setPath); + // API Test 4. Retrieve the set we've just created, validate the response + getSetResponse = given().get(setPath); - logger.info("getSet.getStatusCode(): " + getSet.getStatusCode()); - logger.info("getSet printresponse: " + getSet.prettyPrint()); - assertEquals(200, getSet.getStatusCode()); + System.out.println("getSetResponse.getStatusCode(): " + getSetResponse.getStatusCode()); + System.out.println("getSetResponse, full: " + getSetResponse.prettyPrint()); + assertEquals(200, getSetResponse.getStatusCode()); + + getSetResponse.then().assertThat() + .body("status", equalTo(AbstractApiBean.STATUS_OK)) + .body("data.definition", equalTo("*")) + .body("data.description", equalTo("")) + .body("data.name", equalTo(setName)); + + // API Test 5. Retrieve all sets, check that our new set is listed Response responseAll = given() .get("/api/harvest/server/oaisets"); - logger.info("responseAll.getStatusCode(): " + responseAll.getStatusCode()); - logger.info("responseAll printresponse: " + responseAll.prettyPrint()); + System.out.println("responseAll.getStatusCode(): " + responseAll.getStatusCode()); + System.out.println("responseAll full: " + responseAll.prettyPrint()); assertEquals(200, responseAll.getStatusCode()); - - // try to create set with same name as admin user, should fail - Response r3 = given() + assertTrue(responseAll.body().jsonPath().getList("data.oaisets").size() > 0); + assertTrue(responseAll.body().jsonPath().getList("data.oaisets.name").toString().contains(setName)); // todo: simplify + + // API Test 6. Try to create a set with the same name, should fail + createSetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) .body(jsonForTestSpec(setName, def)) .post(createPath); - assertEquals(400, r3.getStatusCode()); + assertEquals(400, createSetResponse.getStatusCode()); - // try to export set as admin user, should succeed (under admin API, not checking that normal user will fail) + // API Test 7. Try to export set as admin user, should succeed. Set export + // is under /api/admin, no need to try to access it as a non-admin user Response r4 = UtilIT.exportOaiSet(setName); assertEquals(200, r4.getStatusCode()); - - - - // try to delete as normal user, should fail - Response r5 = given() + + // API TEST 8. Try to delete the set as normal user, should fail + Response deleteResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey) .delete(setPath); - logger.info("r5.getStatusCode(): " + r5.getStatusCode()); - assertEquals(400, r5.getStatusCode()); + logger.info("deleteResponse.getStatusCode(): " + deleteResponse.getStatusCode()); + assertEquals(400, deleteResponse.getStatusCode()); - // try to delete as admin user, should work - Response r6 = given() + // API TEST 9. Delete as admin user, should work + deleteResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) .delete(setPath); - logger.info("r6.getStatusCode(): " + r6.getStatusCode()); - assertEquals(200, r6.getStatusCode()); + logger.info("deleteResponse.getStatusCode(): " + deleteResponse.getStatusCode()); + assertEquals(200, deleteResponse.getStatusCode()); } @Test - public void testSetEdit() { - //setupUsers(); + public void testSetEditAPIandOAIlistSets() { + // This test focuses on testing the Edit functionality of the Dataverse + // OAI Set API and the ListSets method of the Dataverse OAI server. + + // Initial setup: crete a test set. + // Since the Create and List (POST and GET) functionality of the API + // is tested extensively in the previous test, we will not be paying + // as much attention to these methods, aside from confirming the + // expected HTTP result codes. + String setName = UtilIT.getRandomString(6); - String def = "*"; + String setDef = "*"; - // make sure the set does not exist - String u0 = String.format("/api/harvest/server/oaisets/%s", setName); + // Make sure the set does not exist + String setPath = String.format("/api/harvest/server/oaisets/%s", setName); String createPath ="/api/harvest/server/oaisets/add"; - Response r0 = given() - .get(u0); - assertEquals(404, r0.getStatusCode()); + Response getSetResponse = given() + .get(setPath); + assertEquals(404, getSetResponse.getStatusCode()); - // try to create set as admin user, should succeed - Response r1 = given() + // Create the set as admin user + Response createSetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) - .body(jsonForTestSpec(setName, def)) + .body(jsonForTestSpec(setName, setDef)) .post(createPath); - assertEquals(201, r1.getStatusCode()); + assertEquals(201, createSetResponse.getStatusCode()); + // I. Test the Modify/Edit (POST method) functionality of the + // Dataverse OAI Sets API - // try to edit as normal user should fail - Response r2 = given() + String newDefinition = "title:New"; + String newDescription = "updated"; + + // API Test 1. Try to modify the set as normal user, should fail + Response editSetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey) - .body(jsonForEditSpec(setName, def,"")) - .put(u0); - logger.info("r2.getStatusCode(): " + r2.getStatusCode()); - assertEquals(400, r2.getStatusCode()); + .body(jsonForEditSpec(setName, setDef, "")) + .put(setPath); + logger.info("non-admin user editSetResponse.getStatusCode(): " + editSetResponse.getStatusCode()); + assertEquals(400, editSetResponse.getStatusCode()); - // try to edit as with blanks should fail - Response r3 = given() + // API Test 2. Try to modify as admin, but with invalid (empty) values, + // should fail + editSetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) .body(jsonForEditSpec(setName, "","")) - .put(u0); - logger.info("r3.getStatusCode(): " + r3.getStatusCode()); - assertEquals(400, r3.getStatusCode()); + .put(setPath); + logger.info("invalid values editSetResponse.getStatusCode(): " + editSetResponse.getStatusCode()); + assertEquals(400, editSetResponse.getStatusCode()); - // try to edit as with something should pass - Response r4 = given() + // API Test 3. Try to modify as admin, with sensible values + editSetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) - .body(jsonForEditSpec(setName, "newDef","newDesc")) - .put(u0); - logger.info("r4 Status code: " + r4.getStatusCode()); - logger.info("r4.prettyPrint(): " + r4.prettyPrint()); - assertEquals(OK.getStatusCode(), r4.getStatusCode()); - - logger.info("u0: " + u0); - // now delete it... - Response r6 = given() + .body(jsonForEditSpec(setName, newDefinition, newDescription)) + .put(setPath); + logger.info("admin user editSetResponse status code: " + editSetResponse.getStatusCode()); + logger.info("admin user editSetResponse.prettyPrint(): " + editSetResponse.prettyPrint()); + assertEquals(OK.getStatusCode(), editSetResponse.getStatusCode()); + + // API Test 4. List the set, confirm that the new values are shown + getSetResponse = given().get(setPath); + + System.out.println("getSetResponse.getStatusCode(): " + getSetResponse.getStatusCode()); + System.out.println("getSetResponse, full: " + getSetResponse.prettyPrint()); + assertEquals(200, getSetResponse.getStatusCode()); + + getSetResponse.then().assertThat() + .body("status", equalTo(AbstractApiBean.STATUS_OK)) + .body("data.definition", equalTo(newDefinition)) + .body("data.description", equalTo(newDescription)) + .body("data.name", equalTo(setName)); + + // II. Test the ListSets functionality of the OAI server + + Response listSetsResponse = UtilIT.getOaiListSets(); + + // 1. Validate the service section of the OAI response: + + XmlPath responseXmlPath = validateOaiVerbResponse(listSetsResponse, "ListSets"); + + // 2. Validate the payload of the response, by confirming that the set + // we created and modified, above, is being listed by the OAI server + // and its xml record is properly formatted + + List listSets = responseXmlPath.getList("OAI-PMH.ListSets.set.list()"); // TODO - maybe try it with findAll()? + assertNotNull(listSets); + assertTrue(listSets.size() > 0); + + Node foundSetNode = null; + for (Node setNode : listSets) { + + if (setName.equals(setNode.get("setName").toString())) { + foundSetNode = setNode; + break; + } + } + + assertNotNull("Newly-created set is not listed by the OAI server", foundSetNode); + assertEquals("Incorrect description in the ListSets entry", newDescription, foundSetNode.getPath("setDescription.metadata.element.field", String.class)); + + // ok, the xml record looks good! + + // Cleanup. Delete the set with the DELETE API + Response deleteSetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) - .delete(u0); - logger.info("r6.getStatusCode(): " + r6.getStatusCode()); - assertEquals(200, r6.getStatusCode()); + .delete(setPath); + assertEquals(200, deleteSetResponse.getStatusCode()); } - // A more elaborate test - we'll create and publish a dataset, then create an - // OAI set with that one dataset, and attempt to retrieve the OAI record - // with GetRecord. + // A more elaborate test - we will create and export an + // OAI set with a single dataset, and attempt to retrieve + // it and validate the OAI server responses of the corresponding + // ListIdentifiers, ListRecords and GetRecord methods. @Test public void testSingleRecordOaiSet() throws InterruptedException { - - //setupUsers(); - - - // Let's try and create an OAI set with the "single set dataset" that // was created as part of the initial setup: @@ -333,12 +389,18 @@ public void testSingleRecordOaiSet() throws InterruptedException { .post(createPath); assertEquals(201, createSetResponse.getStatusCode()); - // TODO: a) look up the set via native harvest/server api; - // b) look up the set via the OAI ListSets; - // export set: - // (this is asynchronous - so we should probably wait a little) + // The GET method of the oai set API, as well as the OAI ListSets + // method are tested extensively in another method in this class, so + // we'll skip checking those here. + + // Let's export the set. This is asynchronous - so we will try to + // wait a little - but in practice, everything potentially time-consuming + // must have been done when the dataset was exported, in the setup method. + Response exportSetResponse = UtilIT.exportOaiSet(setName); assertEquals(200, exportSetResponse.getStatusCode()); + Thread.sleep(1000L); + Response getSet = given() .get(apiPath); @@ -350,25 +412,38 @@ public void testSingleRecordOaiSet() throws InterruptedException { do { - // Run ListIdentifiers on this newly-created set: + // OAI Test 1. Run ListIdentifiers on this newly-created set: Response listIdentifiersResponse = UtilIT.getOaiListIdentifiers(setName, "oai_dc"); - List ret = listIdentifiersResponse.getBody().xmlPath().getList("OAI-PMH.ListIdentifiers.header"); - assertEquals(OK.getStatusCode(), listIdentifiersResponse.getStatusCode()); + + // Validate the service section of the OAI response: + XmlPath responseXmlPath = validateOaiVerbResponse(listIdentifiersResponse, "ListIdentifiers"); + + List ret = responseXmlPath.getList("OAI-PMH.ListIdentifiers.header"); assertNotNull(ret); - logger.info("setName: " + setName); + if (logger.isLoggable(Level.FINE)) { logger.info("listIdentifiersResponse.prettyPrint:..... "); listIdentifiersResponse.prettyPrint(); } - if (ret.size() != 1) { + if (ret.isEmpty()) { + // OK, we'll sleep for another second - provided it's been less + // than 10 sec. total. i++; } else { - // There should be 1 and only 1 record in the response: + // Validate the payload of the ListRecords response: + // a) There should be 1 and only 1 record in the response: assertEquals(1, ret.size()); - // And the record should be the dataset we have just created: - assertEquals(singleSetDatasetPersistentId, listIdentifiersResponse.getBody().xmlPath() + // b) The one record in it should be the dataset we have just created: + assertEquals(singleSetDatasetPersistentId, responseXmlPath .getString("OAI-PMH.ListIdentifiers.header.identifier")); + assertEquals(setName, responseXmlPath + .getString("OAI-PMH.ListIdentifiers.header.setSpec")); + assertNotNull(responseXmlPath.getString("OAI-PMH.ListIdentifiers.header.dateStamp")); + // TODO: validate the formatting of the date string in the record + // header, above! + + // ok, ListIdentifiers response looks valid. break; } Thread.sleep(1000L); @@ -379,34 +454,86 @@ public void testSingleRecordOaiSet() throws InterruptedException { // already happened during its publishing (we made sure to wait there). // Exporting the set should not take any time - but I'll keep that code // in place since it's not going to hurt. - L.A. + System.out.println("Waited " + i + " seconds for OIA export."); //Fail if we didn't find the exported record before the timeout assertTrue(i < maxWait); + + + // OAI Test 2. Run ListRecords, request oai_dc: Response listRecordsResponse = UtilIT.getOaiListRecords(setName, "oai_dc"); assertEquals(OK.getStatusCode(), listRecordsResponse.getStatusCode()); - List listRecords = listRecordsResponse.getBody().xmlPath().getList("OAI-PMH.ListRecords.record"); + + // Validate the service section of the OAI response: + + XmlPath responseXmlPath = validateOaiVerbResponse(listRecordsResponse, "ListRecords"); + + // Validate the payload of the response: + // (the header portion must be identical to that of ListIdentifiers above, + // plus the response must contain a metadata section with a valid oai_dc + // record) + + List listRecords = responseXmlPath.getList("OAI-PMH.ListRecords.record"); + // Same deal, there must be 1 record only in the set: assertNotNull(listRecords); assertEquals(1, listRecords.size()); - assertEquals(singleSetDatasetPersistentId, listRecordsResponse.getBody().xmlPath().getString("OAI-PMH.ListRecords.record[0].header.identifier")); - - // assert that Datacite format does not contain the XML prolog + // a) header section: + assertEquals(singleSetDatasetPersistentId, responseXmlPath.getString("OAI-PMH.ListRecords.record.header.identifier")); + assertEquals(setName, responseXmlPath + .getString("OAI-PMH.ListRecords.record.header.setSpec")); + assertNotNull(responseXmlPath.getString("OAI-PMH.ListRecords.record.header.dateStamp")); + // b) metadata section: + // in the metadata section we are showing the resolver url form of the doi: + String persistentIdUrl = singleSetDatasetPersistentId.replace("doi:", "https://doi.org/"); + assertEquals(persistentIdUrl, responseXmlPath.getString("OAI-PMH.ListRecords.record.metadata.dc.identifier")); + assertEquals("Darwin's Finches", responseXmlPath.getString("OAI-PMH.ListRecords.record.metadata.dc.title")); + assertEquals("Finch, Fiona", responseXmlPath.getString("OAI-PMH.ListRecords.record.metadata.dc.creator")); + assertEquals("Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds.", + responseXmlPath.getString("OAI-PMH.ListRecords.record.metadata.dc.description")); + assertEquals("Medicine, Health and Life Sciences", + responseXmlPath.getString("OAI-PMH.ListRecords.record.metadata.dc.subject")); + // ok, looks legit! + + // OAI Test 3. + // Assert that Datacite format does not contain the XML prolog + // (this is a reference to a resolved issue; generally, harvestable XML + // exports must NOT contain the "")); - // And now run GetRecord on the OAI record for the dataset: - Response getRecordResponse = UtilIT.getOaiRecord(singleSetDatasetPersistentId, "oai_dc"); + // OAI Test 4. run and validate GetRecord response + Response getRecordResponse = UtilIT.getOaiRecord(singleSetDatasetPersistentId, "oai_dc"); System.out.println("GetRecord response in its entirety: "+getRecordResponse.getBody().prettyPrint()); - System.out.println("one more time:"); - getRecordResponse.prettyPrint(); + + // Validate the service section of the OAI response: + responseXmlPath = validateOaiVerbResponse(getRecordResponse, "GetRecord"); + + // Validate the payload of the response: + + // Note that for a set with a single record the output of ListRecrods is + // essentially identical to that of GetRecord! + // (we'll test a multi-record set in a different method) + // a) header section: + assertEquals(singleSetDatasetPersistentId, responseXmlPath.getString("OAI-PMH.GetRecord.record.header.identifier")); + assertEquals(setName, responseXmlPath + .getString("OAI-PMH.GetRecord.record.header.setSpec")); + assertNotNull(responseXmlPath.getString("OAI-PMH.GetRecord.record.header.dateStamp")); + // b) metadata section: + assertEquals(persistentIdUrl, responseXmlPath.getString("OAI-PMH.GetRecord.record.metadata.dc.identifier")); + assertEquals("Darwin's Finches", responseXmlPath.getString("OAI-PMH.GetRecord.record.metadata.dc.title")); + assertEquals("Finch, Fiona", responseXmlPath.getString("OAI-PMH.GetRecord.record.metadata.dc.creator")); + assertEquals("Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds.", + responseXmlPath.getString("OAI-PMH.GetRecord.record.metadata.dc.description")); + assertEquals("Medicine, Health and Life Sciences", responseXmlPath.getString("OAI-PMH.GetRecord.record.metadata.dc.subject")); - assertEquals(singleSetDatasetPersistentId, getRecordResponse.getBody().xmlPath().getString("OAI-PMH.GetRecord.record.header.identifier")); + // ok, looks legit! - // TODO: - // check the actual metadata payload of the OAI record more carefully? } // This test will attempt to create a set with multiple records (enough diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 9fa47db167b..ac767279bd4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2630,6 +2630,11 @@ static Response getOaiListMetadataFormats() { return given().get(oaiVerbPath); } + static Response getOaiListSets() { + String oaiVerbPath = "/oai?verb=ListSets"; + return given().get(oaiVerbPath); + } + static Response getOaiRecord(String datasetPersistentId, String metadataFormat) { String apiPath = String.format("/oai?verb=GetRecord&identifier=%s&metadataPrefix=%s", datasetPersistentId, metadataFormat); return given().get(apiPath); From 9cbfa31d4489ed4ce6df6e37a0fecf92f3a77d18 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 12 Dec 2022 13:51:58 -0500 Subject: [PATCH 05/11] extra (extra tedious) server tests validating paging (resumptionToken) functionality of ListIdentifiers and ListRecords (#8843) --- .../iq/dataverse/api/HarvestingServerIT.java | 340 +++++++++++++++++- .../edu/harvard/iq/dataverse/api/UtilIT.java | 18 +- 2 files changed, 351 insertions(+), 7 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index d25ffd225d9..3497c71e169 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -16,6 +16,8 @@ import static javax.ws.rs.core.Response.Status.OK; import static org.hamcrest.CoreMatchers.equalTo; import java.util.List; +import java.util.Set; +import java.util.HashSet; //import static junit.framework.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -35,6 +37,7 @@ public class HarvestingServerIT { private static String adminUserAPIKey; private static String singleSetDatasetIdentifier; private static String singleSetDatasetPersistentId; + private static List extraDatasetsIdentifiers = new ArrayList<>(); @BeforeClass public static void setUpClass() { @@ -98,6 +101,28 @@ private static void setupDatasets() { // takes longer than just publish/reindex. // So wait for all of this to finish. UtilIT.sleepForReexport(singleSetDatasetPersistentId, adminUserAPIKey, 10); + + // ... And let's create 4 more datasets for a multi-dataset experiment: + + for (int i = 0; i < 4; i++) { + // create dataset: + createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, adminUserAPIKey); + createDatasetResponse.prettyPrint(); + datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + + // retrieve the global id: + String thisDatasetPersistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse); + + // publish dataset: + publishDataset = UtilIT.publishDatasetViaNativeApi(thisDatasetPersistentId, "major", adminUserAPIKey); + assertEquals(200, publishDataset.getStatusCode()); + + UtilIT.sleepForReexport(thisDatasetPersistentId, adminUserAPIKey, 10); + + extraDatasetsIdentifiers.add(thisDatasetPersistentId.substring(thisDatasetPersistentId.lastIndexOf('/') + 1)); + } + + } private String jsonForTestSpec(String name, String def) { @@ -423,16 +448,16 @@ public void testSingleRecordOaiSet() throws InterruptedException { assertNotNull(ret); if (logger.isLoggable(Level.FINE)) { - logger.info("listIdentifiersResponse.prettyPrint:..... "); - listIdentifiersResponse.prettyPrint(); + logger.info("listIdentifiersResponse.prettyPrint: " + + listIdentifiersResponse.prettyPrint()); } if (ret.isEmpty()) { // OK, we'll sleep for another second - provided it's been less // than 10 sec. total. i++; } else { - // Validate the payload of the ListRecords response: - // a) There should be 1 and only 1 record in the response: + // Validate the payload of the ListIdentifiers response: + // a) There should be 1 and only 1 item listed: assertEquals(1, ret.size()); // b) The one record in it should be the dataset we have just created: assertEquals(singleSetDatasetPersistentId, responseXmlPath @@ -537,12 +562,315 @@ public void testSingleRecordOaiSet() throws InterruptedException { } // This test will attempt to create a set with multiple records (enough - // to trigger a paged response with a continuation token) and test its - // performance. + // to trigger a paged respons) and test the resumption token functionality). + // Note that this test requires the OAI service to be configured with some + // non-default settings (the paging limits for ListIdentifiers and ListRecords + // must be set to something low, like 2). @Test public void testMultiRecordOaiSet() throws InterruptedException { + // Setup: Let's create a control OAI set with the 5 datasets created + // in the class init: + + String setName = UtilIT.getRandomString(6); + String setQuery = "(dsPersistentId:" + singleSetDatasetIdentifier; + for (String persistentId : extraDatasetsIdentifiers) { + setQuery = setQuery.concat(" OR dsPersistentId:" + persistentId); + } + setQuery = setQuery.concat(")"); + + String createPath = "/api/harvest/server/oaisets/add"; + + Response createSetResponse = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) + .body(jsonForTestSpec(setName, setQuery)) + .post(createPath); + assertEquals(201, createSetResponse.getStatusCode()); + + // Dataverse OAI Sets API is tested extensively in other methods here, + // so no need to test in any more details than confirming the OK result + // above + Response exportSetResponse = UtilIT.exportOaiSet(setName); + assertEquals(200, exportSetResponse.getStatusCode()); + Thread.sleep(1000L); + + // OAI Test 1. Run ListIdentifiers on the set we've just created: + Response listIdentifiersResponse = UtilIT.getOaiListIdentifiers(setName, "oai_dc"); + assertEquals(OK.getStatusCode(), listIdentifiersResponse.getStatusCode()); + + // Validate the service section of the OAI response: + XmlPath responseXmlPath = validateOaiVerbResponse(listIdentifiersResponse, "ListIdentifiers"); + + List ret = responseXmlPath.getList("OAI-PMH.ListIdentifiers.header.identifier"); + assertNotNull(ret); + + if (logger.isLoggable(Level.FINE)) { + logger.info("listIdentifiersResponse.prettyPrint: "+listIdentifiersResponse.prettyPrint()); + } + + // Validate the payload of the ListIdentifiers response: + // 1a) There should be 2 items listed: + assertEquals("Wrong number of items on the first ListIdentifiers page", + 2, ret.size()); + + // 1b) The response contains a resumptionToken for the next page of items: + String resumptionToken = responseXmlPath.getString("OAI-PMH.ListIdentifiers.resumptionToken"); + assertNotNull("No resumption token in the ListIdentifiers response", resumptionToken); + + // 1c) The total number of items in the set (5) is listed correctly: + assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@completeListSize")); + + // 1d) ... and the offset (cursor) is at the right position (0): + assertEquals(0, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@cursor")); + + // The formatting of individual item records in the ListIdentifiers response + // is tested extensively in the previous test method, so we are not + // looking at them in such detail here; but we should record the + // identifiers listed, so that we can confirm that all the set is + // served as expected. + + Set persistentIdsInListIdentifiers = new HashSet<>(); + + for (String persistentId : ret) { + persistentIdsInListIdentifiers.add(persistentId.substring(persistentId.lastIndexOf('/') + 1)); + } + + // ok, let's move on to the next ListIdentifiers page: + // (we repeat the exact same checks as the above; minus the different + // expected offset) + + // OAI Test 2. Run ListIdentifiers with the resumptionToken obtained + // in the previous step: + + listIdentifiersResponse = UtilIT.getOaiListIdentifiersWithResumptionToken(resumptionToken); + assertEquals(OK.getStatusCode(), listIdentifiersResponse.getStatusCode()); + + // Validate the service section of the OAI response: + responseXmlPath = validateOaiVerbResponse(listIdentifiersResponse, "ListIdentifiers"); + + ret = responseXmlPath.getList("OAI-PMH.ListIdentifiers.header.identifier"); + assertNotNull(ret); + + if (logger.isLoggable(Level.FINE)) { + logger.info("listIdentifiersResponse.prettyPrint: "+listIdentifiersResponse.prettyPrint()); + } + + // Validate the payload of the ListIdentifiers response: + // 2a) There should still be 2 items listed: + assertEquals("Wrong number of items on the second ListIdentifiers page", + 2, ret.size()); + + // 2b) The response should contain a resumptionToken for the next page of items: + resumptionToken = responseXmlPath.getString("OAI-PMH.ListIdentifiers.resumptionToken"); + assertNotNull("No resumption token in the ListIdentifiers response", resumptionToken); + + // 2c) The total number of items in the set (5) is listed correctly: + assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@completeListSize")); + + // 2d) ... and the offset (cursor) is at the right position (2): + assertEquals(2, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@cursor")); + + // Record the identifiers listed on this results page: + + for (String persistentId : ret) { + persistentIdsInListIdentifiers.add(persistentId.substring(persistentId.lastIndexOf('/') + 1)); + } + + // And now the next and the final ListIdentifiers page. + // This time around we should get an *empty* resumptionToken (indicating + // that there are no more results): + + // OAI Test 3. Run ListIdentifiers with the final resumptionToken + + listIdentifiersResponse = UtilIT.getOaiListIdentifiersWithResumptionToken(resumptionToken); + assertEquals(OK.getStatusCode(), listIdentifiersResponse.getStatusCode()); + + // Validate the service section of the OAI response: + responseXmlPath = validateOaiVerbResponse(listIdentifiersResponse, "ListIdentifiers"); + + ret = responseXmlPath.getList("OAI-PMH.ListIdentifiers.header.identifier"); + assertNotNull(ret); + + if (logger.isLoggable(Level.FINE)) { + logger.info("listIdentifiersResponse.prettyPrint: "+listIdentifiersResponse.prettyPrint()); + } + + // Validate the payload of the ListIdentifiers response: + // 3a) There should be only 1 item listed: + assertEquals("Wrong number of items on the final ListIdentifiers page", + 1, ret.size()); + + // 3b) The response contains a resumptionToken for the next page of items: + resumptionToken = responseXmlPath.getString("OAI-PMH.ListIdentifiers.resumptionToken"); + assertNotNull("No resumption token in the final ListIdentifiers response", resumptionToken); + assertTrue("Non-empty resumption token in the final ListIdentifiers response", "".equals(resumptionToken)); + + // 3c) The total number of items in the set (5) is still listed correctly: + assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@completeListSize")); + + // 3d) ... and the offset (cursor) is at the right position (4): + assertEquals(4, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@cursor")); + // Record the last identifier listed on this final page: + persistentIdsInListIdentifiers.add(ret.get(0).substring(ret.get(0).lastIndexOf('/') + 1)); + + // Finally, let's confirm that the expected 5 datasets have been listed + // as part of this Set: + + boolean allDatasetsListed = true; + + allDatasetsListed = persistentIdsInListIdentifiers.contains(singleSetDatasetIdentifier); + for (String persistentId : extraDatasetsIdentifiers) { + allDatasetsListed = persistentIdsInListIdentifiers.contains(persistentId); + } + + assertTrue("Control datasets not properly listed in the paged ListIdentifiers response", + allDatasetsListed); + + // OK, it is safe to assume ListIdentifiers works as it should in page mode. + + // We will now repeat the exact same tests for ListRecords (again, no + // need to pay close attention to the formatting of the individual records, + // since that's tested in the previous test method, since our focus is + // testing the paging/resumptionToken functionality) + + // OAI Test 4. Run ListRecords on the set we've just created: + Response listRecordsResponse = UtilIT.getOaiListRecords(setName, "oai_dc"); + assertEquals(OK.getStatusCode(), listRecordsResponse.getStatusCode()); + + // Validate the service section of the OAI response: + responseXmlPath = validateOaiVerbResponse(listRecordsResponse, "ListRecords"); + + ret = responseXmlPath.getList("OAI-PMH.ListRecords.record.header.identifier"); + assertNotNull(ret); + + if (logger.isLoggable(Level.FINE)) { + logger.info("listRecordsResponse.prettyPrint: "+listRecordsResponse.prettyPrint()); + } + + // Validate the payload of the ListRecords response: + // 4a) There should be 2 items listed: + assertEquals("Wrong number of items on the first ListRecords page", + 2, ret.size()); + + // 4b) The response contains a resumptionToken for the next page of items: + resumptionToken = responseXmlPath.getString("OAI-PMH.ListRecords.resumptionToken"); + assertNotNull("No resumption token in the ListRecords response", resumptionToken); + + // 4c) The total number of items in the set (5) is listed correctly: + assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@completeListSize")); + + // 4d) ... and the offset (cursor) is at the right position (0): + assertEquals(0, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@cursor")); + + Set persistentIdsInListRecords = new HashSet<>(); + + for (String persistentId : ret) { + persistentIdsInListRecords.add(persistentId.substring(persistentId.lastIndexOf('/') + 1)); + } + + // ok, let's move on to the next ListRecords page: + // (we repeat the exact same checks as the above; minus the different + // expected offset) + + // OAI Test 5. Run ListRecords with the resumptionToken obtained + // in the previous step: + + listRecordsResponse = UtilIT.getOaiListRecordsWithResumptionToken(resumptionToken); + assertEquals(OK.getStatusCode(), listRecordsResponse.getStatusCode()); + + // Validate the service section of the OAI response: + responseXmlPath = validateOaiVerbResponse(listRecordsResponse, "ListRecords"); + + ret = responseXmlPath.getList("OAI-PMH.ListRecords.record.header.identifier"); + assertNotNull(ret); + + if (logger.isLoggable(Level.FINE)) { + logger.info("listRecordsResponse.prettyPrint: "+listRecordsResponse.prettyPrint()); + } + + // Validate the payload of the ListRecords response: + // 4a) There should still be 2 items listed: + assertEquals("Wrong number of items on the second ListRecords page", + 2, ret.size()); + + // 4b) The response should contain a resumptionToken for the next page of items: + resumptionToken = responseXmlPath.getString("OAI-PMH.ListRecords.resumptionToken"); + assertNotNull("No resumption token in the ListRecords response", resumptionToken); + + // 4c) The total number of items in the set (5) is listed correctly: + assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@completeListSize")); + + // 4d) ... and the offset (cursor) is at the right position (2): + assertEquals(2, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@cursor")); + + // Record the identifiers listed on this results page: + + for (String persistentId : ret) { + persistentIdsInListRecords.add(persistentId.substring(persistentId.lastIndexOf('/') + 1)); + } + + // And now the next and the final ListRecords page. + // This time around we should get an *empty* resumptionToken (indicating + // that there are no more results): + + // OAI Test 6. Run ListRecords with the final resumptionToken + + listRecordsResponse = UtilIT.getOaiListRecordsWithResumptionToken(resumptionToken); + assertEquals(OK.getStatusCode(), listRecordsResponse.getStatusCode()); + + // Validate the service section of the OAI response: + responseXmlPath = validateOaiVerbResponse(listRecordsResponse, "ListRecords"); + + ret = responseXmlPath.getList("OAI-PMH.ListRecords.record.header.identifier"); + assertNotNull(ret); + + if (logger.isLoggable(Level.FINE)) { + logger.info("listRecordsResponse.prettyPrint: "+listRecordsResponse.prettyPrint()); + } + + // Validate the payload of the ListRecords response: + // 6a) There should be only 1 item listed: + assertEquals("Wrong number of items on the final ListRecords page", + 1, ret.size()); + + // 6b) The response contains a resumptionToken for the next page of items: + resumptionToken = responseXmlPath.getString("OAI-PMH.ListRecords.resumptionToken"); + assertNotNull("No resumption token in the final ListRecords response", resumptionToken); + assertTrue("Non-empty resumption token in the final ListRecords response", "".equals(resumptionToken)); + + // 6c) The total number of items in the set (5) is still listed correctly: + assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@completeListSize")); + + // 6d) ... and the offset (cursor) is at the right position (4): + assertEquals(4, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@cursor")); + + // Record the last identifier listed on this final page: + persistentIdsInListRecords.add(ret.get(0).substring(ret.get(0).lastIndexOf('/') + 1)); + + // Finally, let's confirm that the expected 5 datasets have been listed + // as part of this Set: + + allDatasetsListed = true; + + allDatasetsListed = persistentIdsInListRecords.contains(singleSetDatasetIdentifier); + for (String persistentId : extraDatasetsIdentifiers) { + allDatasetsListed = persistentIdsInListRecords.contains(persistentId); + } + + assertTrue("Control datasets not properly listed in the paged ListRecords response", + allDatasetsListed); + + // OK, it is safe to assume ListRecords works as it should in page mode + // as well. + + // And finally, let's delete the set + String setPath = String.format("/api/harvest/server/oaisets/%s", setName); + Response deleteResponse = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) + .delete(setPath); + logger.info("deleteResponse.getStatusCode(): " + deleteResponse.getStatusCode()); + assertEquals("Failed to delete the control multi-record set", 200, deleteResponse.getStatusCode()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index ac767279bd4..e669a268010 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2641,7 +2641,18 @@ static Response getOaiRecord(String datasetPersistentId, String metadataFormat) } static Response getOaiListIdentifiers(String setName, String metadataFormat) { - String apiPath = String.format("/oai?verb=ListIdentifiers&set=%s&metadataPrefix=%s", setName, metadataFormat); + + String apiPath; + if (StringUtil.nonEmpty(setName)) { + apiPath = String.format("/oai?verb=ListIdentifiers&set=%s&metadataPrefix=%s", setName, metadataFormat); + } else { + apiPath = String.format("/oai?verb=ListIdentifiers&metadataPrefix=%s", metadataFormat); + } + return given().get(apiPath); + } + + static Response getOaiListIdentifiersWithResumptionToken(String resumptionToken) { + String apiPath = String.format("/oai?verb=ListIdentifiers&resumptionToken=%s", resumptionToken); return given().get(apiPath); } @@ -2649,6 +2660,11 @@ static Response getOaiListRecords(String setName, String metadataFormat) { String apiPath = String.format("/oai?verb=ListRecords&set=%s&metadataPrefix=%s", setName, metadataFormat); return given().get(apiPath); } + + static Response getOaiListRecordsWithResumptionToken(String resumptionToken) { + String apiPath = String.format("/oai?verb=ListRecords&resumptionToken=%s", resumptionToken); + return given().get(apiPath); + } static Response changeAuthenticatedUserIdentifier(String oldIdentifier, String newIdentifier, String apiToken) { Response response; From 395d605a8e156dd2ee295a8aa2a0892cad898617 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 12 Dec 2022 17:04:44 -0500 Subject: [PATCH 06/11] An automated test of an actual harvest (#8843) --- .../iq/dataverse/api/HarvestingClients.java | 31 +--- .../iq/dataverse/api/HarvestingClientsIT.java | 169 ++++++++++++++++-- .../iq/dataverse/api/HarvestingServerIT.java | 8 + 3 files changed, 164 insertions(+), 44 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java index 42534514b68..b75cb687c62 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java @@ -373,13 +373,13 @@ public Response startHarvestingJob(@PathParam("nickName") String clientNickname, } if (authenticatedUser == null || !authenticatedUser.isSuperuser()) { - return error(Response.Status.FORBIDDEN, "Only the Dataverse Admin user can run harvesting jobs"); + return error(Response.Status.FORBIDDEN, "Only admin users can run harvesting jobs"); } HarvestingClient harvestingClient = harvestingClientService.findByNickname(clientNickname); if (harvestingClient == null) { - return error(Response.Status.NOT_FOUND, "No such dataverse: "+clientNickname); + return error(Response.Status.NOT_FOUND, "No such client: "+clientNickname); } DataverseRequest dataverseRequest = createDataverseRequest(authenticatedUser); @@ -391,35 +391,8 @@ public Response startHarvestingJob(@PathParam("nickName") String clientNickname, return this.accepted(); } - // This GET shows the status of the harvesting run in progress for this - // client, if present: - // @GET - // @Path("{nickName}/run") - // TODO: - - // This DELETE kills the harvesting run in progress for this client, - // if present: - // @DELETE - // @Path("{nickName}/run") - // TODO: - - - - - /* Auxiliary, helper methods: */ - /* - @Deprecated - public static JsonArrayBuilder harvestingConfigsAsJsonArray(List harvestingDataverses) { - JsonArrayBuilder hdArr = Json.createArrayBuilder(); - - for (Dataverse hd : harvestingDataverses) { - hdArr.add(harvestingConfigAsJson(hd.getHarvestingClientConfig())); - } - return hdArr; - }*/ - public static JsonObjectBuilder harvestingConfigAsJson(HarvestingClient harvestingConfig) { if (harvestingConfig == null) { return null; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java index 9eac3545e54..8fef360c68b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java @@ -1,34 +1,58 @@ package edu.harvard.iq.dataverse.api; import java.util.logging.Logger; +import java.util.logging.Level; import com.jayway.restassured.RestAssured; import static com.jayway.restassured.RestAssured.given; import org.junit.Test; import com.jayway.restassured.response.Response; +import static javax.ws.rs.core.Response.Status.CREATED; +import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; +import static javax.ws.rs.core.Response.Status.ACCEPTED; +import static javax.ws.rs.core.Response.Status.OK; import static org.hamcrest.CoreMatchers.equalTo; -import static junit.framework.Assert.assertEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import org.junit.BeforeClass; /** - * extremely minimal (for now) API tests for creating OAI clients. + * This class tests Harvesting Client functionality. + * Note that these methods test BOTH the proprietary Dataverse rest API for + * creating and managing harvesting clients, AND the underlining OAI-PMH + * harvesting functionality itself. I.e., we will use the Dataverse + * /api/harvest/clients/ api to run an actual harvest of a control set and + * then validate the resulting harvested content. */ public class HarvestingClientsIT { private static final Logger logger = Logger.getLogger(HarvestingClientsIT.class.getCanonicalName()); private static final String harvestClientsApi = "/api/harvest/clients/"; - private static final String harvestCollection = "root"; + private static final String rootCollection = "root"; private static final String harvestUrl = "https://demo.dataverse.org/oai"; private static final String archiveUrl = "https://demo.dataverse.org"; private static final String harvestMetadataFormat = "oai_dc"; private static final String archiveDescription = "RestAssured harvesting client test"; + private static final String controlOaiSet = "controlTestSet"; + private static final int datasetsInControlSet = 7; + private static String normalUserAPIKey; + private static String adminUserAPIKey; + private static String harvestCollectionAlias; @BeforeClass public static void setUpClass() { RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); + + // Create the users, an admin and a non-admin: + setupUsers(); + + // Create a collection that we will use to harvest remote content into: + setupCollection(); + } - private void setupUsers() { + private static void setupUsers() { Response cu0 = UtilIT.createRandomUser(); normalUserAPIKey = UtilIT.getApiTokenFromResponse(cu0); Response cu1 = UtilIT.createRandomUser(); @@ -36,13 +60,22 @@ private void setupUsers() { Response u1a = UtilIT.makeSuperUser(un1); adminUserAPIKey = UtilIT.getApiTokenFromResponse(cu1); } + + private static void setupCollection() { + Response createDataverseResponse = UtilIT.createRandomDataverse(adminUserAPIKey); + createDataverseResponse.prettyPrint(); + assertEquals(CREATED.getStatusCode(), createDataverseResponse.getStatusCode()); + + harvestCollectionAlias = UtilIT.getAliasFromResponse(createDataverseResponse); - private String normalUserAPIKey; - private String adminUserAPIKey; + // publish dataverse: + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(harvestCollectionAlias, adminUserAPIKey); + assertEquals(OK.getStatusCode(), publishDataverse.getStatusCode()); + } @Test public void testCreateEditDeleteClient() { - setupUsers(); + //setupUsers(); String nickName = UtilIT.getRandomString(6); @@ -52,7 +85,7 @@ public void testCreateEditDeleteClient() { + "\"harvestUrl\":\"%s\"," + "\"archiveUrl\":\"%s\"," + "\"metadataFormat\":\"%s\"}", - harvestCollection, harvestUrl, archiveUrl, harvestMetadataFormat); + rootCollection, harvestUrl, archiveUrl, harvestMetadataFormat); // Try to create a client as normal user, should fail: @@ -61,7 +94,7 @@ public void testCreateEditDeleteClient() { .header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey) .body(clientJson) .post(clientApiPath); - assertEquals(401, rCreate.getStatusCode()); + assertEquals(UNAUTHORIZED.getStatusCode(), rCreate.getStatusCode()); // Try to create the same as admin user, should succeed: @@ -70,7 +103,7 @@ public void testCreateEditDeleteClient() { .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) .body(clientJson) .post(clientApiPath); - assertEquals(201, rCreate.getStatusCode()); + assertEquals(CREATED.getStatusCode(), rCreate.getStatusCode()); // Try to update the client we have just created: @@ -80,7 +113,7 @@ public void testCreateEditDeleteClient() { .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) .body(updateJson) .put(clientApiPath); - assertEquals(200, rUpdate.getStatusCode()); + assertEquals(OK.getStatusCode(), rUpdate.getStatusCode()); // Now let's retrieve the client we've just created and edited: @@ -89,7 +122,7 @@ public void testCreateEditDeleteClient() { logger.info("getClient.getStatusCode(): " + getClientResponse.getStatusCode()); logger.info("getClient printresponse: " + getClientResponse.prettyPrint()); - assertEquals(200, getClientResponse.getStatusCode()); + assertEquals(OK.getStatusCode(), getClientResponse.getStatusCode()); // ... and validate the values: @@ -98,7 +131,7 @@ public void testCreateEditDeleteClient() { .body("data.type", equalTo("oai")) .body("data.nickName", equalTo(nickName)) .body("data.archiveDescription", equalTo(archiveDescription)) - .body("data.dataverseAlias", equalTo(harvestCollection)) + .body("data.dataverseAlias", equalTo(rootCollection)) .body("data.harvestUrl", equalTo(harvestUrl)) .body("data.archiveUrl", equalTo(archiveUrl)) .body("data.metadataFormat", equalTo(harvestMetadataFormat)); @@ -109,7 +142,7 @@ public void testCreateEditDeleteClient() { .header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey) .delete(clientApiPath); logger.info("rDelete.getStatusCode(): " + rDelete.getStatusCode()); - assertEquals(401, rDelete.getStatusCode()); + assertEquals(UNAUTHORIZED.getStatusCode(), rDelete.getStatusCode()); // Try to delete as admin user should work: @@ -117,6 +150,112 @@ public void testCreateEditDeleteClient() { .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) .delete(clientApiPath); logger.info("rDelete.getStatusCode(): " + rDelete.getStatusCode()); - assertEquals(200, rDelete.getStatusCode()); + assertEquals(OK.getStatusCode(), rDelete.getStatusCode()); + } + + @Test + public void testHarvestingClientRun() throws InterruptedException { + // This test will create a client and attempt to perform an actual + // harvest and validate the resulting harvested content. + + // Setup: create the client via the API + // since this API is tested somewhat extensively in the previous + // method, we don't need to pay too much attention to this method, aside + // from confirming the expected HTTP status code. + + String nickName = UtilIT.getRandomString(6); + + String clientApiPath = String.format(harvestClientsApi+"%s", nickName); + String clientJson = String.format("{\"dataverseAlias\":\"%s\"," + + "\"type\":\"oai\"," + + "\"harvestUrl\":\"%s\"," + + "\"archiveUrl\":\"%s\"," + + "\"set\":\"%s\"," + + "\"metadataFormat\":\"%s\"}", + harvestCollectionAlias, harvestUrl, archiveUrl, controlOaiSet, harvestMetadataFormat); + + Response createResponse = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) + .body(clientJson) + .post(clientApiPath); + assertEquals(CREATED.getStatusCode(), createResponse.getStatusCode()); + + // API TEST 1. Run the harvest using the configuration (client) we have + // just created + + String runHarvestApiPath = String.format(harvestClientsApi+"%s/run", nickName); + + // TODO? - verify that a non-admin user cannot perform this operation (401) + + Response runResponse = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) + .post(runHarvestApiPath); + assertEquals(ACCEPTED.getStatusCode(), runResponse.getStatusCode()); + + // API TEST 2. As indicated by the ACCEPTED status code above, harvesting + // is an asynchronous operation that will be performed in the background. + // Verify that this "in progress" status is properly reported while it's + // running, and that it completes in some reasonable amount of time. + + int i = 0; + int maxWait=20; // a very conservative interval; this harvest has no business taking this long + do { + // keep checking the status of the client with the GET api: + Response getClientResponse = given() + .get(clientApiPath); + + assertEquals(OK.getStatusCode(), getClientResponse.getStatusCode()); + assertEquals(AbstractApiBean.STATUS_OK, getClientResponse.body().jsonPath().getString("status")); + + if (logger.isLoggable(Level.FINE)) { + logger.info("listIdentifiersResponse.prettyPrint: " + + getClientResponse.prettyPrint()); + } + + String clientStatus = getClientResponse.body().jsonPath().getString("data.status"); + assertNotNull(clientStatus); + + if ("inProgress".equals(clientStatus)) { + // we'll sleep for another second + i++; + } else { + // Check the values in the response: + // a) Confirm that the harvest has completed: + assertEquals("Unexpected client status: "+clientStatus, "inActive", clientStatus); + + // b) Confirm that it has actually succeeded: + assertEquals("Last harvest not reported a success", "SUCCESS", getClientResponse.body().jsonPath().getString("data.lastResult")); + String harvestTimeStamp = getClientResponse.body().jsonPath().getString("data.lastHarvest"); + assertNotNull(harvestTimeStamp); + + // c) Confirm that the other timestamps match: + assertEquals(harvestTimeStamp, getClientResponse.body().jsonPath().getString("data.lastSuccessful")); + assertEquals(harvestTimeStamp, getClientResponse.body().jsonPath().getString("data.lastNonEmpty")); + + // d) Confirm that the correct number of datasets have been harvested: + assertEquals(datasetsInControlSet, getClientResponse.body().jsonPath().getInt("data.lastDatasetsHarvested")); + + // ok, it looks like the harvest has completed successfully. + break; + } + Thread.sleep(1000L); + } while (i Date: Mon, 12 Dec 2022 17:10:35 -0500 Subject: [PATCH 07/11] comments (#8843) --- .../iq/dataverse/api/HarvestingClientsIT.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java index 8fef360c68b..448faa20b0b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java @@ -75,7 +75,9 @@ private static void setupCollection() { @Test public void testCreateEditDeleteClient() { - //setupUsers(); + // This method focuses on testing the native Dataverse harvesting client + // API. + String nickName = UtilIT.getRandomString(6); @@ -158,7 +160,7 @@ public void testHarvestingClientRun() throws InterruptedException { // This test will create a client and attempt to perform an actual // harvest and validate the resulting harvested content. - // Setup: create the client via the API + // Setup: create the client via native API // since this API is tested somewhat extensively in the previous // method, we don't need to pay too much attention to this method, aside // from confirming the expected HTTP status code. @@ -246,8 +248,11 @@ public void testHarvestingClientRun() throws InterruptedException { // Fail if it hasn't completed in maxWait seconds assertTrue(i < maxWait); - // TODO: use the native Dataverses/Datasets apis to verify that the expected - // datasets have been harvested. + // TODO(?) use the native Dataverses/Datasets apis to verify that the expected + // datasets have been harvested. This may or may not be necessary, seeing + // how we have already confirmed the number of successfully harvested + // datasets from the control set; somewhat hard to imagine a practical + // situation where that would not be enough (?). // Cleanup: delete the client From 8e310c35801d5ce6c1f033236d2b588d6ef1f9a9 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 13 Dec 2022 13:54:24 -0500 Subject: [PATCH 08/11] logic! (#8843) --- .../iq/dataverse/api/HarvestingServerIT.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index b5563c926e5..dad32bcaa60 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -565,7 +565,11 @@ public void testSingleRecordOaiSet() throws InterruptedException { // to trigger a paged respons) and test the resumption token functionality). // Note that this test requires the OAI service to be configured with some // non-default settings (the paging limits for ListIdentifiers and ListRecords - // must be set to something low, like 2). + // must be set to 2, in order to be able to trigger this paging behavior without + // having to create and export too many datasets). + // So you will need to do this: + // asadmin create-jvm-options "-Ddataverse.oai.server.maxidentifiers=2" + // asadmin create-jvm-options "-Ddataverse.oai.server.maxrecords=2" @Test @@ -616,7 +620,7 @@ public void testMultiRecordOaiSet() throws InterruptedException { // 1b) The response contains a resumptionToken for the next page of items: String resumptionToken = responseXmlPath.getString("OAI-PMH.ListIdentifiers.resumptionToken"); - assertNotNull("No resumption token in the ListIdentifiers response", resumptionToken); + assertNotNull("No resumption token in the ListIdentifiers response (has the jvm option dataverse.oai.server.maxidentifiers been configured?)", resumptionToken); // 1c) The total number of items in the set (5) is listed correctly: assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@completeListSize")); @@ -722,7 +726,7 @@ public void testMultiRecordOaiSet() throws InterruptedException { allDatasetsListed = persistentIdsInListIdentifiers.contains(singleSetDatasetIdentifier); for (String persistentId : extraDatasetsIdentifiers) { - allDatasetsListed = persistentIdsInListIdentifiers.contains(persistentId); + allDatasetsListed = allDatasetsListed && persistentIdsInListIdentifiers.contains(persistentId); } assertTrue("Control datasets not properly listed in the paged ListIdentifiers response", @@ -756,7 +760,7 @@ public void testMultiRecordOaiSet() throws InterruptedException { // 4b) The response contains a resumptionToken for the next page of items: resumptionToken = responseXmlPath.getString("OAI-PMH.ListRecords.resumptionToken"); - assertNotNull("No resumption token in the ListRecords response", resumptionToken); + assertNotNull("No resumption token in the ListRecords response (has the jvm option dataverse.oai.server.maxrecords been configured?)", resumptionToken); // 4c) The total number of items in the set (5) is listed correctly: assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@completeListSize")); @@ -856,7 +860,7 @@ public void testMultiRecordOaiSet() throws InterruptedException { allDatasetsListed = persistentIdsInListRecords.contains(singleSetDatasetIdentifier); for (String persistentId : extraDatasetsIdentifiers) { - allDatasetsListed = persistentIdsInListRecords.contains(persistentId); + allDatasetsListed = allDatasetsListed && persistentIdsInListRecords.contains(persistentId); } assertTrue("Control datasets not properly listed in the paged ListRecords response", @@ -879,6 +883,6 @@ public void testMultiRecordOaiSet() throws InterruptedException { // Some ideas: // - Test handling of deleted dataset records // - Test "from" and "until" time parameters - // - Test validating full verb response records against XML schema + // - Validate full verb response records against XML schema // (for each supported metadata format, possibly?) } From b5986fa94c954de72f4280e1e9bde81ed9389910 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 13 Dec 2022 16:44:38 -0500 Subject: [PATCH 09/11] cleanup (#8843) --- .../harvest/client/FastGetRecord.java | 2 +- .../harvest/client/HarvesterServiceBean.java | 2 +- .../iq/dataverse/api/HarvestingClientsIT.java | 66 +++++++------- .../iq/dataverse/api/HarvestingServerIT.java | 88 ++++++++----------- 4 files changed, 73 insertions(+), 85 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java index 5b3e4df331d..c5e3a93e2df 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java @@ -130,7 +130,7 @@ public void harvestRecord(String baseURL, String identifier, String metadataPref int responseCode = 0; con = (HttpURLConnection) url.openConnection(); - con.setRequestProperty("User-Agent", "DataverseHarvester/3.0"); + con.setRequestProperty("User-Agent", "Dataverse Harvesting Client v5"); con.setRequestProperty("Accept-Encoding", "compress, gzip, identify"); try { diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java index e7156dfe9aa..a0c52e4b80c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java @@ -372,7 +372,7 @@ File retrieveProprietaryDataverseMetadata (HttpClient client, String remoteApiUr HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(remoteApiUrl)) .GET() - .header("User-Agent", "DataverseHarvester/6.0") + .header("User-Agent", "Dataverse Harvesting Client v5") .build(); HttpResponse response; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java index 448faa20b0b..d9b4d502f59 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java @@ -4,6 +4,7 @@ import java.util.logging.Level; import com.jayway.restassured.RestAssured; import static com.jayway.restassured.RestAssured.given; +import com.jayway.restassured.path.json.JsonPath; import org.junit.Test; import com.jayway.restassured.response.Response; import static javax.ws.rs.core.Response.Status.CREATED; @@ -28,14 +29,14 @@ public class HarvestingClientsIT { private static final Logger logger = Logger.getLogger(HarvestingClientsIT.class.getCanonicalName()); - private static final String harvestClientsApi = "/api/harvest/clients/"; - private static final String rootCollection = "root"; - private static final String harvestUrl = "https://demo.dataverse.org/oai"; - private static final String archiveUrl = "https://demo.dataverse.org"; - private static final String harvestMetadataFormat = "oai_dc"; - private static final String archiveDescription = "RestAssured harvesting client test"; - private static final String controlOaiSet = "controlTestSet"; - private static final int datasetsInControlSet = 7; + private static final String HARVEST_CLIENTS_API = "/api/harvest/clients/"; + private static final String ROOT_COLLECTION = "root"; + private static final String HARVEST_URL = "https://demo.dataverse.org/oai"; + private static final String ARCHIVE_URL = "https://demo.dataverse.org"; + private static final String HARVEST_METADATA_FORMAT = "oai_dc"; + private static final String ARCHIVE_DESCRIPTION = "RestAssured harvesting client test"; + private static final String CONTROL_OAI_SET = "controlTestSet"; + private static final int DATASETS_IN_CONTROL_SET = 7; private static String normalUserAPIKey; private static String adminUserAPIKey; private static String harvestCollectionAlias; @@ -81,13 +82,13 @@ public void testCreateEditDeleteClient() { String nickName = UtilIT.getRandomString(6); - String clientApiPath = String.format(harvestClientsApi+"%s", nickName); + String clientApiPath = String.format(HARVEST_CLIENTS_API+"%s", nickName); String clientJson = String.format("{\"dataverseAlias\":\"%s\"," + "\"type\":\"oai\"," + "\"harvestUrl\":\"%s\"," + "\"archiveUrl\":\"%s\"," + "\"metadataFormat\":\"%s\"}", - rootCollection, harvestUrl, archiveUrl, harvestMetadataFormat); + ROOT_COLLECTION, HARVEST_URL, ARCHIVE_URL, HARVEST_METADATA_FORMAT); // Try to create a client as normal user, should fail: @@ -109,7 +110,7 @@ public void testCreateEditDeleteClient() { // Try to update the client we have just created: - String updateJson = String.format("{\"archiveDescription\":\"%s\"}", archiveDescription); + String updateJson = String.format("{\"archiveDescription\":\"%s\"}", ARCHIVE_DESCRIPTION); Response rUpdate = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) @@ -132,11 +133,11 @@ public void testCreateEditDeleteClient() { .body("status", equalTo(AbstractApiBean.STATUS_OK)) .body("data.type", equalTo("oai")) .body("data.nickName", equalTo(nickName)) - .body("data.archiveDescription", equalTo(archiveDescription)) - .body("data.dataverseAlias", equalTo(rootCollection)) - .body("data.harvestUrl", equalTo(harvestUrl)) - .body("data.archiveUrl", equalTo(archiveUrl)) - .body("data.metadataFormat", equalTo(harvestMetadataFormat)); + .body("data.archiveDescription", equalTo(ARCHIVE_DESCRIPTION)) + .body("data.dataverseAlias", equalTo(ROOT_COLLECTION)) + .body("data.harvestUrl", equalTo(HARVEST_URL)) + .body("data.archiveUrl", equalTo(ARCHIVE_URL)) + .body("data.metadataFormat", equalTo(HARVEST_METADATA_FORMAT)); // Try to delete the client as normal user should fail: @@ -167,14 +168,14 @@ public void testHarvestingClientRun() throws InterruptedException { String nickName = UtilIT.getRandomString(6); - String clientApiPath = String.format(harvestClientsApi+"%s", nickName); + String clientApiPath = String.format(HARVEST_CLIENTS_API+"%s", nickName); String clientJson = String.format("{\"dataverseAlias\":\"%s\"," + "\"type\":\"oai\"," + "\"harvestUrl\":\"%s\"," + "\"archiveUrl\":\"%s\"," + "\"set\":\"%s\"," + "\"metadataFormat\":\"%s\"}", - harvestCollectionAlias, harvestUrl, archiveUrl, controlOaiSet, harvestMetadataFormat); + harvestCollectionAlias, HARVEST_URL, ARCHIVE_URL, CONTROL_OAI_SET, HARVEST_METADATA_FORMAT); Response createResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) @@ -185,7 +186,7 @@ public void testHarvestingClientRun() throws InterruptedException { // API TEST 1. Run the harvest using the configuration (client) we have // just created - String runHarvestApiPath = String.format(harvestClientsApi+"%s/run", nickName); + String runHarvestApiPath = String.format(HARVEST_CLIENTS_API+"%s/run", nickName); // TODO? - verify that a non-admin user cannot perform this operation (401) @@ -207,35 +208,36 @@ public void testHarvestingClientRun() throws InterruptedException { .get(clientApiPath); assertEquals(OK.getStatusCode(), getClientResponse.getStatusCode()); - assertEquals(AbstractApiBean.STATUS_OK, getClientResponse.body().jsonPath().getString("status")); + JsonPath responseJsonPath = getClientResponse.body().jsonPath(); + assertNotNull("Invalid JSON in GET client response", responseJsonPath); + assertEquals(AbstractApiBean.STATUS_OK, responseJsonPath.getString("status")); - if (logger.isLoggable(Level.FINE)) { - logger.info("listIdentifiersResponse.prettyPrint: " - + getClientResponse.prettyPrint()); - } - - String clientStatus = getClientResponse.body().jsonPath().getString("data.status"); + String clientStatus = responseJsonPath.getString("data.status"); assertNotNull(clientStatus); - if ("inProgress".equals(clientStatus)) { + if ("inProgress".equals(clientStatus) || "IN PROGRESS".equals(responseJsonPath.getString("data.lastResult"))) { // we'll sleep for another second i++; } else { + if (logger.isLoggable(Level.FINE)) { + logger.info("getClientResponse.prettyPrint: " + + getClientResponse.prettyPrint()); + } // Check the values in the response: // a) Confirm that the harvest has completed: assertEquals("Unexpected client status: "+clientStatus, "inActive", clientStatus); // b) Confirm that it has actually succeeded: - assertEquals("Last harvest not reported a success", "SUCCESS", getClientResponse.body().jsonPath().getString("data.lastResult")); - String harvestTimeStamp = getClientResponse.body().jsonPath().getString("data.lastHarvest"); + assertEquals("Last harvest not reported a success", "SUCCESS", responseJsonPath.getString("data.lastResult")); + String harvestTimeStamp = responseJsonPath.getString("data.lastHarvest"); assertNotNull(harvestTimeStamp); // c) Confirm that the other timestamps match: - assertEquals(harvestTimeStamp, getClientResponse.body().jsonPath().getString("data.lastSuccessful")); - assertEquals(harvestTimeStamp, getClientResponse.body().jsonPath().getString("data.lastNonEmpty")); + assertEquals(harvestTimeStamp, responseJsonPath.getString("data.lastSuccessful")); + assertEquals(harvestTimeStamp, responseJsonPath.getString("data.lastNonEmpty")); // d) Confirm that the correct number of datasets have been harvested: - assertEquals(datasetsInControlSet, getClientResponse.body().jsonPath().getInt("data.lastDatasetsHarvested")); + assertEquals(DATASETS_IN_CONTROL_SET, responseJsonPath.getInt("data.lastDatasetsHarvested")); // ok, it looks like the harvest has completed successfully. break; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index dad32bcaa60..d10e0c4c6d7 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -18,7 +18,6 @@ import java.util.List; import java.util.Set; import java.util.HashSet; -//import static junit.framework.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -141,9 +140,12 @@ private XmlPath validateOaiVerbResponse(Response oaiResponse, String verb) { assertNotNull(responseXmlPath); String dateString = responseXmlPath.getString("OAI-PMH.responseDate"); - assertNotNull(dateString); // TODO: validate that it's well-formatted! - logger.info("date string from the OAI output:"+dateString); - assertEquals("http://localhost:8080/oai", responseXmlPath.getString("OAI-PMH.request")); + assertNotNull(dateString); + // TODO: validate the formatting of the date string in the record + // header, above. (could be slightly tricky - since this formatting + // is likely locale-specific) + logger.fine("date string from the OAI output:"+dateString); + //assertEquals("http://localhost:8080/oai", responseXmlPath.getString("OAI-PMH.request")); assertEquals(verb, responseXmlPath.getString("OAI-PMH.request.@verb")); return responseXmlPath; } @@ -153,12 +155,11 @@ public void testOaiIdentify() { // Run Identify: Response identifyResponse = UtilIT.getOaiIdentify(); assertEquals(OK.getStatusCode(), identifyResponse.getStatusCode()); - //logger.info("Identify response: "+identifyResponse.prettyPrint()); // Validate the response: XmlPath responseXmlPath = validateOaiVerbResponse(identifyResponse, "Identify"); - assertEquals("http://localhost:8080/oai", responseXmlPath.getString("OAI-PMH.Identify.baseURL")); + //assertEquals("http://localhost:8080/oai", responseXmlPath.getString("OAI-PMH.Identify.baseURL")); // Confirm that the server is reporting the correct parameters that // our server implementation should be using: assertEquals("2.0", responseXmlPath.getString("OAI-PMH.Identify.protocolVersion")); @@ -171,7 +172,6 @@ public void testOaiListMetadataFormats() { // Run ListMeatadataFormats: Response listFormatsResponse = UtilIT.getOaiListMetadataFormats(); assertEquals(OK.getStatusCode(), listFormatsResponse.getStatusCode()); - //logger.info("ListMetadataFormats response: "+listFormatsResponse.prettyPrint()); // Validate the response: @@ -253,7 +253,7 @@ public void testNativeSetAPI() { System.out.println("responseAll full: " + responseAll.prettyPrint()); assertEquals(200, responseAll.getStatusCode()); assertTrue(responseAll.body().jsonPath().getList("data.oaisets").size() > 0); - assertTrue(responseAll.body().jsonPath().getList("data.oaisets.name").toString().contains(setName)); // todo: simplify + assertTrue(responseAll.body().jsonPath().getList("data.oaisets.name", String.class).contains(setName)); // API Test 6. Try to create a set with the same name, should fail createSetResponse = given() @@ -369,22 +369,14 @@ public void testSetEditAPIandOAIlistSets() { // we created and modified, above, is being listed by the OAI server // and its xml record is properly formatted - List listSets = responseXmlPath.getList("OAI-PMH.ListSets.set.list()"); // TODO - maybe try it with findAll()? - assertNotNull(listSets); - assertTrue(listSets.size() > 0); - - Node foundSetNode = null; - for (Node setNode : listSets) { - - if (setName.equals(setNode.get("setName").toString())) { - foundSetNode = setNode; - break; - } - } + List listSets = responseXmlPath.getList("OAI-PMH.ListSets.set.list().findAll{it.setName=='"+setName+"'}", Node.class); + + // 2a. Confirm that our set is listed: + assertNotNull("Unexpected response from ListSets", listSets); + assertTrue("Newly-created set isn't properly listed by the OAI server", listSets.size() == 1); + // 2b. Confirm that the set entry contains the updated description: + assertEquals("Incorrect description in the ListSets entry", newDescription, listSets.get(0).getPath("setDescription.metadata.element.field", String.class)); - assertNotNull("Newly-created set is not listed by the OAI server", foundSetNode); - assertEquals("Incorrect description in the ListSets entry", newDescription, foundSetNode.getPath("setDescription.metadata.element.field", String.class)); - // ok, the xml record looks good! // Cleanup. Delete the set with the DELETE API @@ -416,26 +408,30 @@ public void testSingleRecordOaiSet() throws InterruptedException { // The GET method of the oai set API, as well as the OAI ListSets // method are tested extensively in another method in this class, so - // we'll skip checking those here. + // we'll skip looking too closely into those here. - // Let's export the set. This is asynchronous - so we will try to - // wait a little - but in practice, everything potentially time-consuming - // must have been done when the dataset was exported, in the setup method. + // A quick test that the new set is listed under native API + Response getSet = given() + .get(apiPath); + assertEquals(200, getSet.getStatusCode()); + + // Export the set. Response exportSetResponse = UtilIT.exportOaiSet(setName); assertEquals(200, exportSetResponse.getStatusCode()); - Thread.sleep(1000L); - - Response getSet = given() - .get(apiPath); + + // Strictly speaking, exporting an OAI set is an asynchronous operation. + // So the code below was written to expect to have to wait for up to 10 + // additional seconds for it to complete. In retrospect, this is + // most likely unnecessary (because the only potentially expensive part + // of the process is the metadata export, and in this case that must have + // already happened - when the dataset was published (that operation + // now has its own wait mechanism). But I'll keep this extra code in + // place since it's not going to hurt. - L.A. - logger.info("getSet.getStatusCode(): " + getSet.getStatusCode()); - logger.fine("getSet printresponse: " + getSet.prettyPrint()); - assertEquals(200, getSet.getStatusCode()); int i = 0; int maxWait=10; do { - // OAI Test 1. Run ListIdentifiers on this newly-created set: Response listIdentifiersResponse = UtilIT.getOaiListIdentifiers(setName, "oai_dc"); @@ -445,17 +441,14 @@ public void testSingleRecordOaiSet() throws InterruptedException { XmlPath responseXmlPath = validateOaiVerbResponse(listIdentifiersResponse, "ListIdentifiers"); List ret = responseXmlPath.getList("OAI-PMH.ListIdentifiers.header"); - assertNotNull(ret); - if (logger.isLoggable(Level.FINE)) { - logger.info("listIdentifiersResponse.prettyPrint: " - + listIdentifiersResponse.prettyPrint()); - } - if (ret.isEmpty()) { - // OK, we'll sleep for another second - provided it's been less - // than 10 sec. total. + if (ret == null || ret.isEmpty()) { + // OK, we'll sleep for another second i++; } else { + if (logger.isLoggable(Level.FINE)) { + logger.info("listIdentifiersResponse.prettyPrint: " + listIdentifiersResponse.prettyPrint()); + } // Validate the payload of the ListIdentifiers response: // a) There should be 1 and only 1 item listed: assertEquals(1, ret.size()); @@ -465,20 +458,13 @@ public void testSingleRecordOaiSet() throws InterruptedException { assertEquals(setName, responseXmlPath .getString("OAI-PMH.ListIdentifiers.header.setSpec")); assertNotNull(responseXmlPath.getString("OAI-PMH.ListIdentifiers.header.dateStamp")); - // TODO: validate the formatting of the date string in the record - // header, above! + // TODO: validate the formatting of the date string here as well. // ok, ListIdentifiers response looks valid. break; } Thread.sleep(1000L); } while (i Date: Tue, 13 Dec 2022 19:43:44 -0500 Subject: [PATCH 10/11] one more jenkins run, with a bit more logging (#8843) --- .../edu/harvard/iq/dataverse/api/HarvestingClientsIT.java | 6 ++---- .../edu/harvard/iq/dataverse/api/HarvestingServerIT.java | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java index d9b4d502f59..3fc72125145 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java @@ -219,10 +219,8 @@ public void testHarvestingClientRun() throws InterruptedException { // we'll sleep for another second i++; } else { - if (logger.isLoggable(Level.FINE)) { - logger.info("getClientResponse.prettyPrint: " - + getClientResponse.prettyPrint()); - } + logger.info("getClientResponse.prettyPrint: " + + getClientResponse.prettyPrint()); // Check the values in the response: // a) Confirm that the harvest has completed: assertEquals("Unexpected client status: "+clientStatus, "inActive", clientStatus); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index d10e0c4c6d7..ccc0629bb69 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -429,6 +429,7 @@ public void testSingleRecordOaiSet() throws InterruptedException { // now has its own wait mechanism). But I'll keep this extra code in // place since it's not going to hurt. - L.A. + Thread.sleep(1000L); // initial sleep interval int i = 0; int maxWait=10; do { From f6d08bb2fcef119b39a3e3a193dc5826026ea7a9 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 14 Dec 2022 10:55:12 -0500 Subject: [PATCH 11/11] trigger another Jenkins run, with the time delay slightly rearranged in the wait for an async. operation + some extra logging (#8843) --- .../edu/harvard/iq/dataverse/api/HarvestingClientsIT.java | 8 ++++++-- .../edu/harvard/iq/dataverse/api/HarvestingServerIT.java | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java index 3fc72125145..094eb0df77c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java @@ -203,6 +203,11 @@ public void testHarvestingClientRun() throws InterruptedException { int i = 0; int maxWait=20; // a very conservative interval; this harvest has no business taking this long do { + // Give it an initial 1 sec. delay, to make sure the client state + // has been updated in the database, which can take some appreciable + // amount of time on a heavily-loaded server running a full suite of + // tests: + Thread.sleep(1000L); // keep checking the status of the client with the GET api: Response getClientResponse = given() .get(clientApiPath); @@ -226,7 +231,7 @@ public void testHarvestingClientRun() throws InterruptedException { assertEquals("Unexpected client status: "+clientStatus, "inActive", clientStatus); // b) Confirm that it has actually succeeded: - assertEquals("Last harvest not reported a success", "SUCCESS", responseJsonPath.getString("data.lastResult")); + assertEquals("Last harvest not reported a success (took "+i+" seconds)", "SUCCESS", responseJsonPath.getString("data.lastResult")); String harvestTimeStamp = responseJsonPath.getString("data.lastHarvest"); assertNotNull(harvestTimeStamp); @@ -240,7 +245,6 @@ public void testHarvestingClientRun() throws InterruptedException { // ok, it looks like the harvest has completed successfully. break; } - Thread.sleep(1000L); } while (i