From 96b810d2f8125dea7dd93678b0bb6c7ea01119a5 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 5 Dec 2019 17:21:40 -0500 Subject: [PATCH 1/7] #6300 Add more detail to dataset search results --- .../harvard/iq/dataverse/DatasetVersion.java | 58 +++++++++++ .../iq/dataverse/search/SolrSearchResult.java | 99 ++++++++++++++++++- 2 files changed, 156 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index 48479872c63..98ffea94adf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -1130,6 +1130,62 @@ public List getKeywords() { return getCompoundChildFieldValues(DatasetFieldConstant.keyword, DatasetFieldConstant.keywordValue); } + public List getRelatedMaterial() { + List relMaterial = new ArrayList<>(); + for (DatasetField dsf : this.getDatasetFields()) { + if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.relatedMaterial)) { + relMaterial.addAll(dsf.getValues()); + } + } + return relMaterial; + } + + public List getDataSource() { + List dataSources = new ArrayList<>(); + for (DatasetField dsf : this.getDatasetFields()) { + if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.dataSources)) { + dataSources.addAll(dsf.getValues()); + } + } + return dataSources; + } + + public List getGeographicCoverage() { + List geoCoverages = new ArrayList<>(); + + for (DatasetField dsf : this.getDatasetFields()) { + if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.geographicCoverage)) { + for (DatasetFieldCompoundValue geoCoverage : dsf.getDatasetFieldCompoundValues()) { + String country = null; + String state = null; + String city = null; + String other = null; + String[] coverageItem = null; + for (DatasetField subField : geoCoverage.getChildDatasetFields()) { + if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.country)) { + country = subField.getDisplayValue(); + } + if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.state)) { + state = subField.getDisplayValue(); + } + if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.city)) { + city = subField.getDisplayValue(); + } + if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.otherGeographicCoverage)) { + other = subField.getDisplayValue(); + } + + coverageItem = new String[]{country, state, city, other}; + } + geoCoverages.add(coverageItem); + } + + } + } + return geoCoverages; + } + + public List getRelatedPublications() { List relatedPublications = new ArrayList<>(); for (DatasetField dsf : this.getDatasetFields()) { @@ -1141,6 +1197,8 @@ public List getRelatedPublications() { String citation = subField.getDisplayValue(); relatedPublication.setText(citation); } + + if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.publicationURL)) { // We have to avoid using subField.getDisplayValue() here - because the DisplayFormatType // for this url metadata field is likely set up so that the display value is automatically diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java index 85681bf0c9f..7d159d7ecb2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java @@ -2,6 +2,8 @@ import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetRelPublication; +import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.api.Util; import edu.harvard.iq.dataverse.dataset.DatasetThumbnail; @@ -437,7 +439,7 @@ public JsonObjectBuilder getJsonForMyData() { .add("parentName", this.getParent().get("name")); } } - + return myDataJson; } //getJsonForMydata @@ -457,6 +459,8 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool String filePersistentId = null; String preferredUrl = null; String apiUrl = null; + String publisherName = null; + if (this.type.equals(SearchConstants.DATAVERSES)) { displayName = this.name; @@ -466,6 +470,8 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool displayName = this.title; identifierLabel = "global_id"; preferredUrl = getPersistentUrl(); + publisherName = this.parent.get("name"); + // if /** * @todo Should we show the name of the parent dataverse? */ @@ -534,6 +540,7 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool .add("file_persistent_id", this.filePersistentId) .add("dataset_name", datasetName) .add("dataset_id", datasetId) + .add("publisher", publisherName) .add("dataset_persistent_id", datasetPersistentId) .add("dataset_citation", datasetCitation) .add("deaccession_reason", this.deaccessionReason) @@ -552,6 +559,96 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool } } + if (this.entity == null) { + + } else { + if (this.entity.isInstanceofDataset()) { + System.out.print("Instance of Dataset: " + this.entity.getStorageIdentifier()); + nullSafeJsonBuilder.add("storageIdentifier", this.entity.getStorageIdentifier()); + Dataset ds = (Dataset) this.entity; + DatasetVersion dv; + if (this.isDraftState()) { + dv = ds.getLatestVersion(); + } else { + dv = ds.getReleasedVersion(); + } + + if (!dv.getKeywords().isEmpty()) { + JsonArrayBuilder keyWords = Json.createArrayBuilder(); + for (String keyword : dv.getKeywords()) { + keyWords.add(keyword); + } + nullSafeJsonBuilder.add("keywords", keyWords); + } + JsonArrayBuilder subjects = Json.createArrayBuilder(); + for (String subject : dv.getDatasetSubjects()) { + subjects.add(subject); + } + nullSafeJsonBuilder.add("subjects", subjects); + nullSafeJsonBuilder.add("versionId", dv.getId()); + nullSafeJsonBuilder.add("versionState", dv.getVersionState().toString()); + nullSafeJsonBuilder.add("createdAt", ds.getCreateDate()); + nullSafeJsonBuilder.add("updatedAt", ds.getModificationTime()); + + if (!dv.getDatasetContacts().isEmpty()) { + JsonArrayBuilder contacts = Json.createArrayBuilder(); + NullSafeJsonBuilder nullSafeJsonBuilderInner = jsonObjectBuilder(); + for (String contact[] : dv.getDatasetContacts()) { + nullSafeJsonBuilderInner.add("name", contact[0]); + nullSafeJsonBuilderInner.add("affiliation", contact[1]); + contacts.add(nullSafeJsonBuilderInner); + } + nullSafeJsonBuilder.add("contacts", contacts); + } + if(!dv.getRelatedPublications().isEmpty()){ + JsonArrayBuilder relPub = Json.createArrayBuilder(); + NullSafeJsonBuilder inner = jsonObjectBuilder(); + for (DatasetRelPublication dsRelPub : dv.getRelatedPublications()) { + inner.add("title", dsRelPub.getTitle()); + inner.add("citation", dsRelPub.getText()); + inner.add("url", dsRelPub.getUrl()); + relPub.add(inner); + } + nullSafeJsonBuilder.add("publications", relPub); + } + + if (!dv.getDatasetProducers().isEmpty()) { + JsonArrayBuilder producers = Json.createArrayBuilder(); + for (String[] producer : dv.getDatasetProducers()) { + producers.add(producer[0]); + } + nullSafeJsonBuilder.add("producers", producers); + } + if (!dv.getRelatedMaterial().isEmpty()) { + JsonArrayBuilder relatedMaterials = Json.createArrayBuilder(); + for (String relatedMaterial : dv.getRelatedMaterial()) { + relatedMaterials.add(relatedMaterial); + } + nullSafeJsonBuilder.add("relatedMaterial", relatedMaterials); + } + + if (!dv.getGeographicCoverage().isEmpty()) { + JsonArrayBuilder geoCov = Json.createArrayBuilder(); + NullSafeJsonBuilder inner = jsonObjectBuilder(); + for (String ind[] : dv.getGeographicCoverage()) { + inner.add("country", ind[0]); + inner.add("state", ind[1]); + inner.add("city", ind[2]); + inner.add("other", ind[3]); + geoCov.add(inner); + } + nullSafeJsonBuilder.add("geographicCoverage", geoCov); + } + if (!dv.getDataSource().isEmpty()) { + JsonArrayBuilder dataSources = Json.createArrayBuilder(); + for (String dsource : dv.getDataSource()) { + dataSources.add(dsource); + } + nullSafeJsonBuilder.add("dataSources", dataSources); + } + } + } + if (showApiUrls) { /** * @todo We should probably have a metadata_url or api_url concept From ad3accfe6ae7ed4e11429a57288c02c71990d1a5 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 6 Dec 2019 10:31:48 -0500 Subject: [PATCH 2/7] #6300 add Version Numbers to Search API --- src/main/java/edu/harvard/iq/dataverse/api/Search.java | 2 +- .../edu/harvard/iq/dataverse/search/SolrSearchResult.java | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Search.java b/src/main/java/edu/harvard/iq/dataverse/api/Search.java index fdd4a671fbd..836a69e351f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Search.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Search.java @@ -136,7 +136,7 @@ public Response search( paginationStart, dataRelatedToMe, numResultsPerPage, - queryEntities + true //SEK get query entities always for search API 12/6/2019 ); } catch (SearchException ex) { Throwable cause = ex; diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java index 7d159d7ecb2..219447536c2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java @@ -563,7 +563,6 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool } else { if (this.entity.isInstanceofDataset()) { - System.out.print("Instance of Dataset: " + this.entity.getStorageIdentifier()); nullSafeJsonBuilder.add("storageIdentifier", this.entity.getStorageIdentifier()); Dataset ds = (Dataset) this.entity; DatasetVersion dv; @@ -587,6 +586,10 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool nullSafeJsonBuilder.add("subjects", subjects); nullSafeJsonBuilder.add("versionId", dv.getId()); nullSafeJsonBuilder.add("versionState", dv.getVersionState().toString()); + if(this.isPublishedState()){ + nullSafeJsonBuilder.add("majorVersion", dv.getVersionNumber()); + nullSafeJsonBuilder.add("minorVersion", dv.getMinorVersionNumber()); + } nullSafeJsonBuilder.add("createdAt", ds.getCreateDate()); nullSafeJsonBuilder.add("updatedAt", ds.getModificationTime()); From b379994e4b319300f9a752006b0dacc2eb1b3c22 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 6 Dec 2019 13:56:20 -0500 Subject: [PATCH 3/7] #6300 Add unit tests for additional data provided --- .../edu/harvard/iq/dataverse/api/Search.java | 2 +- .../harvard/iq/dataverse/api/SearchIT.java | 70 +++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Search.java b/src/main/java/edu/harvard/iq/dataverse/api/Search.java index 836a69e351f..3a49385c0f6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Search.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Search.java @@ -136,7 +136,7 @@ public Response search( paginationStart, dataRelatedToMe, numResultsPerPage, - true //SEK get query entities always for search API 12/6/2019 + true //SEK get query entities always for search API additional Dataset Information 6300 12/6/2019 ); } catch (SearchException ex) { Throwable cause = ex; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index 7cb363e1116..4768a8b80c4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -27,6 +27,7 @@ import static java.lang.Thread.sleep; import static javax.ws.rs.core.Response.Status.CREATED; import static javax.ws.rs.core.Response.Status.NOT_FOUND; +import static javax.ws.rs.core.Response.Status.OK; import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; import org.hamcrest.Matchers; import org.junit.After; @@ -200,7 +201,76 @@ public void testSearchCitation() { assertEquals(200, deleteUserResponse.getStatusCode()); } + + @Test + public void testAdditionalDatasetContent6300() { + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + + Response datasetAsJson = UtilIT.nativeGet(datasetId, apiToken); + datasetAsJson.then().assertThat() + .statusCode(OK.getStatusCode()); + + String identifier = JsonPath.from(datasetAsJson.getBody().asString()).getString("data.identifier"); + + Response getDatasetJsonBeforePublishing = UtilIT.nativeGet(datasetId, apiToken); + getDatasetJsonBeforePublishing.prettyPrint(); + String protocol = JsonPath.from(getDatasetJsonBeforePublishing.getBody().asString()).getString("data.protocol"); + String authority = JsonPath.from(getDatasetJsonBeforePublishing.getBody().asString()).getString("data.authority"); + + String datasetPersistentId = protocol + ":" + authority + "/" + identifier; + String pathToJsonFile = "doc/sphinx-guides/source/_static/api/dataset-add-metadata.json"; + Response addSubjectViaNative = UtilIT.addDatasetMetadataViaNative(datasetPersistentId, pathToJsonFile, apiToken); + addSubjectViaNative.prettyPrint(); + addSubjectViaNative.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response searchResponse = UtilIT.search("id:dataset_" + datasetId + "_draft", apiToken); + searchResponse.prettyPrint(); + /*["Astronomy and Astrophysics"]*/ + assertTrue(searchResponse.body().jsonPath().getString("data.items[0].subjects").contains("Astronomy and Astrophysics")); + assertTrue(searchResponse.body().jsonPath().getString("data.items[0].versionState").equals("DRAFT")); + /* "versionState": "DRAFT",*/ + + //We now need to publish to see version number + Response publishDataverse = UtilIT.publishDataverseViaSword(dataverseAlias, apiToken); + publishDataverse.prettyPrint(); + publishDataverse.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken); + publishDataset.prettyPrint(); + publishDataset.then().assertThat() + .statusCode(OK.getStatusCode()); + + searchResponse = UtilIT.search("id:dataset_" + datasetId, apiToken); + searchResponse.prettyPrint(); + /*["Astronomy and Astrophysics"]*/ + assertTrue(searchResponse.body().jsonPath().getString("data.items[0].subjects").contains("Astronomy and Astrophysics")); + assertTrue(searchResponse.body().jsonPath().getString("data.items[0].versionState").equals("RELEASED")); + + assertTrue(searchResponse.body().jsonPath().getString("data.items[0].majorVersion").equals("1")); + assertTrue(searchResponse.body().jsonPath().getString("data.items[0].minorVersion").equals("0")); + + assertTrue(searchResponse.body().jsonPath().getString("data.items[0].authors").contains("Spruce, Sabrina")); + + assertTrue(searchResponse.body().jsonPath().getString("data.items[0].contacts[0].name").contains("Finch, Fiona")); + assertTrue(searchResponse.body().jsonPath().getString("data.items[0].storageIdentifier").contains(identifier)); + + } + + /* * Note: this test does a lot of checking for permissions with / without privlidged api key. * Thumbnails access is the same with/without that access as of 4.9.4 --MAD From dba46e845d0697e6fc9c34e193f8f4d2f776b3e9 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 11 Dec 2019 10:06:36 -0500 Subject: [PATCH 4/7] 6300 remove reference to parameter --- doc/sphinx-guides/source/api/search.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index f3d474d1b90..c0a26ec7a35 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -35,7 +35,6 @@ show_relevance boolean Whether or not to show details of which fields were ma show_facets boolean Whether or not to show facets that can be operated on by the "fq" parameter. False by default. See :ref:`advanced search example `. fq string A filter query on the search term. Multiple "fq" parameters can be used. See :ref:`advanced search example `. show_entity_ids boolean Whether or not to show the database IDs of the search results (for developer use). -query_entities boolean Whether entities are queried via direct database calls (for developer use). =============== ======= =========== Basic Search Example From a33bcdd348fa4541419752ed6a95c1bfe619b471 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 11 Dec 2019 10:34:16 -0500 Subject: [PATCH 5/7] 6300 Adding example of additional DS info --- doc/sphinx-guides/source/api/search.rst | 46 +++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index c0a26ec7a35..a35a544596e 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -48,7 +48,7 @@ https://demo.dataverse.org/api/search?q=trees "status":"OK", "data":{ "q":"trees", - "total_count":4, + "total_count":5, "start":0, "spelling_alternatives":{ "trees":"[tree]" @@ -98,9 +98,49 @@ https://demo.dataverse.org/api/search?q=trees "identifier":"birds", "description":"A bird dataverse with some trees", "published_at":"2016-05-10T12:57:27Z" - } + }, + { + "name":"Darwin's Finches", + "type":"dataset", + "url":"https://doi.org/10.70122/FK2/MB5VGR", + "global_id":"doi:10.70122/FK2/MB5VGR", + "description":"Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds.", + "published_at":"2019-12-11T15:26:10Z", + "publisher":"dvbe69f5e1", + "citationHtml":"Finch, Fiona; Spruce, Sabrina; Poe, Edgar Allen; Mulligan, Hercules, 2019, \"Darwin's Finches\", https://doi.org/10.70122/FK2/MB5VGR, Root, V3", + "identifier_of_dataverse":"dvbe69f5e1", + "name_of_dataverse":"dvbe69f5e1", + "citation":"Finch, Fiona; Spruce, Sabrina; Poe, Edgar Allen; Mulligan, Hercules, 2019, \"Darwin's Finches\", https://doi.org/10.70122/FK2/MB5VGR, Root, V3", + "storageIdentifier":"file://10.70122/FK2/MB5VGR", + "subjects":[ + "Astronomy and Astrophysics", + "Other" + ], + "versionId":1260, + "versionState":"RELEASED", + "majorVersion":3, + "minorVersion":0, + "createdAt":"2019-09-20T18:08:29Z", + "updatedAt":"2019-12-11T15:26:10Z", + "contacts":[ + { + "name":"Finch, Fiona", + "affiliation":"" + } + ], + "producers":[ + "Allen, Irwin", + "Spielberg, Stephen" + ], + "authors":[ + "Finch, Fiona", + "Spruce, Sabrina", + "Poe, Edgar Allen", + "Mulligan, Hercules" + ] + } ], - "count_in_response":4 + "count_in_response":5 } } From 1278522b87c6a8fdd832e9eb2409dfd5f5e75648 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 11 Dec 2019 11:03:52 -0500 Subject: [PATCH 6/7] add release note about `query_entities` #6300 #6396 --- doc/release-notes/6300-6396-search-api.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/release-notes/6300-6396-search-api.md diff --git a/doc/release-notes/6300-6396-search-api.md b/doc/release-notes/6300-6396-search-api.md new file mode 100644 index 00000000000..b3caf7c88f8 --- /dev/null +++ b/doc/release-notes/6300-6396-search-api.md @@ -0,0 +1,7 @@ +## Notes for Tool Developers and Integrators + +### Search API + +The boolean parameter `query_entities` has been removed from the Search API. + +The former "true" behavior of "whether entities are queried via direct database calls (for developer use)" is now always true. From 6851c28a365766327cda0d9c42d91be82f806503 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 11 Dec 2019 11:10:09 -0500 Subject: [PATCH 7/7] add a use case to release note #6300 #6396 --- doc/release-notes/6300-6396-search-api.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/release-notes/6300-6396-search-api.md b/doc/release-notes/6300-6396-search-api.md index b3caf7c88f8..f015eeff01c 100644 --- a/doc/release-notes/6300-6396-search-api.md +++ b/doc/release-notes/6300-6396-search-api.md @@ -1,3 +1,9 @@ +## Major Use Cases + +Newly-supported use cases in this release include: + +- Search API users will see additional fields in the JSON output #6300 #6396 + ## Notes for Tool Developers and Integrators ### Search API