From f7938581a7cf7395a44d9dbb777b49ff56be2a02 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 12 Aug 2022 09:12:32 -0400 Subject: [PATCH 1/3] #8868 update handling of undefined mdl --- .../iq/dataverse/DvObjectContainer.java | 3 +++ .../dataverse/export/DublinCoreExporter.java | 2 +- .../iq/dataverse/export/ddi/DdiExportUtil.java | 18 +++++------------- .../iq/dataverse/util/bagit/OREMap.java | 2 +- .../iq/dataverse/util/json/JsonPrinter.java | 9 ++++++--- 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java index 746efded48b..62d8f68467d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java +++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java @@ -93,6 +93,9 @@ public void setMetadataLanguage(String ml) { } } + public static boolean isMetadataLanguageSet(String mdLang) { + return mdLang!=null && !mdLang.equals(UNDEFINED_METADATA_LANGUAGE_CODE); + } /* Dataverse collections can be configured to allow use of Curation labels and have this inheritable value to decide which set of labels to use. diff --git a/src/main/java/edu/harvard/iq/dataverse/export/DublinCoreExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/DublinCoreExporter.java index 7c4ebfdd44d..113e669f511 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/DublinCoreExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/DublinCoreExporter.java @@ -18,7 +18,7 @@ public class DublinCoreExporter implements Exporter { - + @Override public String getProviderName() { return "oai_dc"; diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 1952acb67a3..4bbcd653ac3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -129,7 +129,7 @@ private static void dtoddi(DatasetDTO datasetDto, OutputStream outputStream) thr xmlw.writeAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); xmlw.writeAttribute("xsi:schemaLocation", DDIExporter.DEFAULT_XML_NAMESPACE + " " + DDIExporter.DEFAULT_XML_SCHEMALOCATION); writeAttribute(xmlw, "version", DDIExporter.DEFAULT_XML_VERSION); - if(isMetadataLanguageSet(datasetDto.getMetadataLanguage())) { + if(DvObjectContainer.isMetadataLanguageSet(datasetDto.getMetadataLanguage())) { writeAttribute(xmlw, "xml:lang", datasetDto.getMetadataLanguage()); } createStdyDscr(xmlw, datasetDto); @@ -151,7 +151,7 @@ public static void datasetJson2ddi(JsonObject datasetDtoAsJson, DatasetVersion v xmlw.writeAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); xmlw.writeAttribute("xsi:schemaLocation", DDIExporter.DEFAULT_XML_NAMESPACE + " " + DDIExporter.DEFAULT_XML_SCHEMALOCATION); writeAttribute(xmlw, "version", DDIExporter.DEFAULT_XML_VERSION); - if(isMetadataLanguageSet(datasetDto.getMetadataLanguage())) { + if(DvObjectContainer.isMetadataLanguageSet(datasetDto.getMetadataLanguage())) { writeAttribute(xmlw, "xml:lang", datasetDto.getMetadataLanguage()); } createStdyDscr(xmlw, datasetDto); @@ -161,14 +161,6 @@ public static void datasetJson2ddi(JsonObject datasetDtoAsJson, DatasetVersion v xmlw.writeEndElement(); // codeBook xmlw.flush(); } - - - private static boolean isMetadataLanguageSet(String mdLang) { - if(mdLang!=null && !mdLang.equals(DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE)) { - return true; - } - return false; - } /** * @todo This is just a stub, copied from DDIExportServiceBean. It should @@ -944,7 +936,7 @@ private static void writeDistributorsElement(XMLStreamWriter xmlw, DatasetVersio } if (!distributorName.isEmpty()) { xmlw.writeStartElement("distrbtr"); - if(isMetadataLanguageSet(lang)) { + if(DvObjectContainer.isMetadataLanguageSet(lang)) { writeAttribute(xmlw, "xml:lang", lang); } if (!distributorAffiliation.isEmpty()) { @@ -1064,7 +1056,7 @@ private static void writeAbstractElement(XMLStreamWriter xmlw, DatasetVersionDTO if(!descriptionDate.isEmpty()){ writeAttribute(xmlw,"date",descriptionDate); } - if(isMetadataLanguageSet(lang)) { + if(DvObjectContainer.isMetadataLanguageSet(lang)) { writeAttribute(xmlw, "xml:lang", lang); } xmlw.writeCharacters(descriptionText); @@ -1538,7 +1530,7 @@ private static void writeFullElement (XMLStreamWriter xmlw, String name, String //For the simplest Elements we can if (!StringUtilisEmpty(value)) { xmlw.writeStartElement(name); - if(isMetadataLanguageSet(lang)) { + if(DvObjectContainer.isMetadataLanguageSet(lang)) { writeAttribute(xmlw, "xml:lang", lang); } xmlw.writeCharacters(value); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index a295f264d66..c7363a2ee62 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -219,7 +219,7 @@ public JsonObjectBuilder getOREMapBuilder(boolean aggregationOnly) throws Except aggBuilder.add(JsonLDTerm.schemaOrg("isPartOf").getLabel(), getDataverseDescription(dataset.getOwner())); String mdl = dataset.getMetadataLanguage(); - if(!mdl.equals(DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE)) { + if (DvObjectContainer.isMetadataLanguageSet(mdl)) { aggBuilder.add(JsonLDTerm.schemaOrg("inLanguage").getLabel(), mdl); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 91f1ac2cfbc..49ca2dea3b4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -314,7 +314,7 @@ public static JsonObjectBuilder json(BuiltinUser user) { } public static JsonObjectBuilder json(Dataset ds) { - return jsonObjectBuilder() + JsonObjectBuilder bld = jsonObjectBuilder() .add("id", ds.getId()) .add("identifier", ds.getIdentifier()) .add("persistentUrl", ds.getPersistentURL()) @@ -322,8 +322,11 @@ public static JsonObjectBuilder json(Dataset ds) { .add("authority", ds.getAuthority()) .add("publisher", BrandingUtil.getInstallationBrandName()) .add("publicationDate", ds.getPublicationDateFormattedYYYYMMDD()) - .add("storageIdentifier", ds.getStorageIdentifier()) - .add("metadataLanguage", ds.getMetadataLanguage()); + .add("storageIdentifier", ds.getStorageIdentifier()); + if (DvObjectContainer.isMetadataLanguageSet(ds.getMetadataLanguage())) { + bld.add("metadataLanguage", ds.getMetadataLanguage()); + } + return bld; } public static JsonObjectBuilder json(DatasetVersion dsv) { From 99e158f29677e25a839633405b70ad53d63a2a05 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 24 Aug 2022 11:17:47 -0400 Subject: [PATCH 2/3] #8868 add a release note for export all --- doc/release-notes/8868-fix-json-import.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/release-notes/8868-fix-json-import.md diff --git a/doc/release-notes/8868-fix-json-import.md b/doc/release-notes/8868-fix-json-import.md new file mode 100644 index 00000000000..d054b93da7f --- /dev/null +++ b/doc/release-notes/8868-fix-json-import.md @@ -0,0 +1,5 @@ +The dataset export has been updated to eliminate the export of "undefined" as a metadata language. + +Run ReExportall to update Exports + +Following the directions in the [Admin Guide](http://guides.dataverse.org/en/5.12/admin/metadataexport.html#batch-exports-through-the-api) From 4fdc499d2beaf07901c0646fdef03024a3df7fbf Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 26 Aug 2022 18:00:58 -0400 Subject: [PATCH 3/3] per feedback, adding a quick mention of the duplicated keyword entry under "bugfixes" (#8868) --- doc/release-notes/8868-fix-json-import.md | 4 +++- .../java/edu/harvard/iq/dataverse/api/DataversesIT.java | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/8868-fix-json-import.md b/doc/release-notes/8868-fix-json-import.md index d054b93da7f..de0366e395e 100644 --- a/doc/release-notes/8868-fix-json-import.md +++ b/doc/release-notes/8868-fix-json-import.md @@ -1,4 +1,6 @@ -The dataset export has been updated to eliminate the export of "undefined" as a metadata language. +Under "bug fixes": + +Small bugs have been fixed in the dataset export in the JSON and DDI formats; eliminating the export of "undefined" as a metadata language in the former, and a duplicate keyword tag in the latter. Run ReExportall to update Exports diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 5f25d2eb68d..95c2d27dc5c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -486,7 +486,7 @@ public void testDataFileAPIPermissions() { } @Test - public void testImportDDI() throws IOException { + public void testImportDDI() throws IOException, InterruptedException { Response createUser = UtilIT.createRandomUser(); String username = UtilIT.getUsernameFromResponse(createUser); @@ -534,6 +534,12 @@ public void testImportDDI() throws IOException { Response destroyDatasetResponsePidRel = UtilIT.destroyDataset(datasetIdIntPidRel, apiToken); assertEquals(200, destroyDatasetResponsePidRel.getStatusCode()); + // This last dataset we have just imported, let's give it a sec. to finish indexing (?) + // or whatever it is that may still be happening. (Have been seeing intermittent 500 from the next + // destroyDataset() line lately) + + Thread.sleep(1000L); + Integer datasetIdIntRelease = JsonPath.from(importDDIRelease.body().asString()).getInt("data.id"); Response destroyDatasetResponseRelease = UtilIT.destroyDataset(datasetIdIntRelease, apiToken); assertEquals(200, destroyDatasetResponseRelease.getStatusCode());