SPO/6751 cvv i18n

doc/sphinx-guides/source/installation/config.rst

@@ -688,7 +688,7 @@ When creating or editing a dataset, users will be asked to enter the metadata in
 
 ``curl http://localhost:8080/api/admin/settings/:MetadataLanguages -X PUT -d '[{"locale":"en","title":"English"},{"locale":"fr","title":"Français"}]'``
 
-Note that metadata selected from Controlled Vocabularies will also display in the metadata language of the dataset, but only if translations have been configured, i.e. you configure the "lang" directory and populate it with translations as described below).
+Note that metadata selected from Controlled Vocabularies will also display in the metadata language of the dataset, but only if translations have been configured, i.e. you configure the "lang" directory and populate it with translations as described below). In metadata export files, controlled vocabulary values will be included in the Dataverse installations default language and in the metadata language of the dataset (if specified).
 
 Configuring the "lang" Directory
 ++++++++++++++++++++++++++++++++

src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java

@@ -140,6 +140,7 @@ public void setOptionSelectItems(List<SelectItem> optionSelectItems) {
 
     public DatasetFieldType() {}
 
+    //For use in tests
     public DatasetFieldType(String name, FieldType fieldType, boolean allowMultiples) {
         // use the name for both default name and title
         this.name = name;

src/main/java/edu/harvard/iq/dataverse/api/dto/MetadataBlockDTO.java

@@ -71,7 +71,7 @@ public void setFields(List<FieldDTO> fields) {
 
         @Override
         public String toString() {
-            return "MetadataBlockDTO{" + "displayName=" + displayName + ", fields=" + fields + '}';
+            return "MetadataBlockDTO{" + "displayName=" + displayName + ", name=" + name + ", fields=" + fields + '}';
         }
 
         public String getName() {

src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java

@@ -1,6 +1,8 @@
 package edu.harvard.iq.dataverse.export.ddi;
 
 import com.google.gson.Gson;
+
+import edu.harvard.iq.dataverse.ControlledVocabularyValue;
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.DataTable;
 import edu.harvard.iq.dataverse.DatasetFieldConstant;
@@ -260,14 +262,14 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto)
         // Study Info
         xmlw.writeStartElement("stdyInfo");
 
-        writeSubjectElement(xmlw, version); //Subject and Keywords
+        writeSubjectElement(xmlw, version, datasetDto.getMetadataLanguage()); //Subject and Keywords
         writeAbstractElement(xmlw, version, datasetDto.getMetadataLanguage()); // Description
-        writeSummaryDescriptionElement(xmlw, version);
+        writeSummaryDescriptionElement(xmlw, version, datasetDto.getMetadataLanguage());
         writeFullElement(xmlw, "notes", dto2Primitive(version, DatasetFieldConstant.notesText));
         ////////
         xmlw.writeEndElement(); // stdyInfo
 
-        writeMethodElement(xmlw, version);
+        writeMethodElement(xmlw, version, datasetDto.getMetadataLanguage());
         writeDataAccess(xmlw , version);
         writeOtherStudyMaterial(xmlw , version);
 
@@ -380,7 +382,7 @@ private static void writeVersionStatement(XMLStreamWriter xmlw, DatasetVersionDT
         xmlw.writeEndElement(); // verStmt
     }
 
-    private static void writeSummaryDescriptionElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException {
+    private static void writeSummaryDescriptionElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO, String lang) throws XMLStreamException {
         xmlw.writeStartElement("sumDscr");
         for (Map.Entry<String, MetadataBlockDTO> entry : datasetVersionDTO.getMetadataBlocks().entrySet()) {
             String key = entry.getKey();
@@ -510,7 +512,7 @@ private static void writeSummaryDescriptionElement(XMLStreamWriter xmlw, Dataset
                         writeMultipleElement(xmlw, "universe", fieldDTO);
                     }
                     if (DatasetFieldConstant.unitOfAnalysis.equals(fieldDTO.getTypeName())) {
-                        writeMultipleElement(xmlw, "anlyUnit", fieldDTO);                     
+                        writeI18NElementList(xmlw, "anlyUnit", fieldDTO.getMultipleVocab(), "unitOfAnalysis", fieldDTO.getTypeClass(), "socialscience", lang);
                     }
                 }              
             }
@@ -535,14 +537,14 @@ private static void writeDateElement(XMLStreamWriter xmlw, String element, Strin
 
     }
 
-    private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO version) throws XMLStreamException{
+    private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO version, String lang) throws XMLStreamException{
         xmlw.writeStartElement("method");
         xmlw.writeStartElement("dataColl");
-        writeFullElement(xmlw, "timeMeth", dto2Primitive(version, DatasetFieldConstant.timeMethod)); 
+        writeI18NElement(xmlw, "timeMeth", version, DatasetFieldConstant.timeMethod,lang); 
         writeFullElement(xmlw, "dataCollector", dto2Primitive(version, DatasetFieldConstant.dataCollector));         
         writeFullElement(xmlw, "collectorTraining", dto2Primitive(version, DatasetFieldConstant.collectorTraining));   
         writeFullElement(xmlw, "frequenc", dto2Primitive(version, DatasetFieldConstant.frequencyOfDataCollection));      
-        writeFullElement(xmlw, "sampProc", dto2Primitive(version, DatasetFieldConstant.samplingProcedure));
+        writeI18NElement(xmlw, "sampProc", version, DatasetFieldConstant.samplingProcedure, lang);
 
         writeTargetSampleElement(xmlw, version);
 
@@ -555,8 +557,8 @@ private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO
         writeFullElement(xmlw, "srcDocu", dto2Primitive(version, DatasetFieldConstant.accessToSources));
         xmlw.writeEndElement(); //sources
 
-        writeFullElement(xmlw, "collMode", dto2Primitive(version, DatasetFieldConstant.collectionMode)); 
-        writeFullElement(xmlw, "resInstru", dto2Primitive(version, DatasetFieldConstant.researchInstrument)); 
+        writeI18NElement(xmlw, "collMode", version, DatasetFieldConstant.collectionMode, lang); 
+        writeI18NElement(xmlw, "resInstru", version, DatasetFieldConstant.researchInstrument, lang); 
         writeFullElement(xmlw, "collSitu", dto2Primitive(version, DatasetFieldConstant.dataCollectionSituation)); 
         writeFullElement(xmlw, "actMin", dto2Primitive(version, DatasetFieldConstant.actionsToMinimizeLoss));
         writeFullElement(xmlw, "conOps", dto2Primitive(version, DatasetFieldConstant.controlOperations));
@@ -575,7 +577,7 @@ private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO
         xmlw.writeEndElement();//method
     }
 
-    private static void writeSubjectElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException{ 
+    private static void writeSubjectElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO, String lang) throws XMLStreamException{ 
 
         //Key Words and Topic Classification
 
@@ -586,11 +588,7 @@ private static void writeSubjectElement(XMLStreamWriter xmlw, DatasetVersionDTO
             if ("citation".equals(key)) {
                 for (FieldDTO fieldDTO : value.getFields()) {
                     if (DatasetFieldConstant.subject.equals(fieldDTO.getTypeName())){
-                        for ( String subject : fieldDTO.getMultipleVocab()){
-                            xmlw.writeStartElement("keyword");
-                            xmlw.writeCharacters(subject);
-                            xmlw.writeEndElement(); //Keyword
-                        }
+                        writeI18NElementList(xmlw, "keyword", fieldDTO.getMultipleVocab(), "subject", fieldDTO.getTypeClass(), "citation", lang);
                     }
 
                     if (DatasetFieldConstant.keyword.equals(fieldDTO.getTypeName())) {
@@ -1333,6 +1331,22 @@ private static String dto2Primitive(DatasetVersionDTO datasetVersionDTO, String
         return null;
     }
 
+    private static String dto2Primitive(DatasetVersionDTO datasetVersionDTO, String datasetFieldTypeName, Locale locale) {
+        for (Map.Entry<String, MetadataBlockDTO> entry : datasetVersionDTO.getMetadataBlocks().entrySet()) {
+            MetadataBlockDTO value = entry.getValue();
+            for (FieldDTO fieldDTO : value.getFields()) {
+                if (datasetFieldTypeName.equals(fieldDTO.getTypeName())) {
+                    String rawVal = fieldDTO.getSinglePrimitive();
+                    if (fieldDTO.getTypeClass().equals("controlledVocabulary")) {
+                        return ControlledVocabularyValue.getLocaleStrValue(rawVal, datasetFieldTypeName, value.getName(),
+                                locale, false);
+                    }
+                }
+            }
+        }
+        return null;
+    }
+
     private static List<String> dto2PrimitiveList(DatasetVersionDTO datasetVersionDTO, String datasetFieldTypeName) {
         for (Map.Entry<String, MetadataBlockDTO> entry : datasetVersionDTO.getMetadataBlocks().entrySet()) {
             MetadataBlockDTO value = entry.getValue();
@@ -1355,6 +1369,71 @@ private static void writeFullElementList(XMLStreamWriter xmlw, String name, List
             }
         }
     }
+
+    private static void writeI18NElementList(XMLStreamWriter xmlw, String name, List<String> values,
+            String fieldTypeName, String fieldTypeClass, String metadataBlockName, String lang)
+            throws XMLStreamException {
+
+        if (values != null && !values.isEmpty()) {
+            Locale defaultLocale = Locale.getDefault();
+            for (String value : values) {
+                if (fieldTypeClass.equals("controlledVocabulary")) {
+                    String localeVal = ControlledVocabularyValue.getLocaleStrValue(value, fieldTypeName, metadataBlockName, defaultLocale, false);
+                    if (localeVal != null) {
+
+                        value = localeVal;
+                        writeFullElement(xmlw, name, value, defaultLocale.getLanguage());
+                    } else {
+                        writeFullElement(xmlw, name, value);
+                    }
+                } else {
+                    writeFullElement(xmlw, name, value);
+                }
+            }
+            if (lang != null && !defaultLocale.getLanguage().equals(lang)) {
+                // Get values in dataset metadata language
+                // Loop before testing fieldTypeClass to be ready for external CVV
+                for (String value : values) {
+                    if (fieldTypeClass.equals("controlledVocabulary")) {
+                        String localeVal = ControlledVocabularyValue.getLocaleStrValue(value, fieldTypeName, metadataBlockName, new Locale(lang), false);
+                        if (localeVal != null) {
+                            writeFullElement(xmlw, name, localeVal, lang);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    private static void writeI18NElement(XMLStreamWriter xmlw, String name, DatasetVersionDTO version,
+            String fieldTypeName, String lang) throws XMLStreamException {
+        // Get the default value
+        String val = dto2Primitive(version, fieldTypeName);
+        Locale defaultLocale = Locale.getDefault();
+        // Get the language-specific value for the default language
+        String localeVal = dto2Primitive(version, fieldTypeName, defaultLocale);
+        String requestedLocaleVal = null;
+        if (lang != null && !defaultLocale.getLanguage().equals(lang)) {
+            // Also get the value in the requested locale/lang if that's not the default
+            // lang.
+            requestedLocaleVal = dto2Primitive(version, fieldTypeName, new Locale(lang));
+        }
+        // FWIW locale-specific vals will only be non-null for CVV values (at present)
+        if (localeVal == null && requestedLocaleVal == null) {
+            // Not CVV/no translations so print without lang tag
+            writeFullElement(xmlw, name, val);
+        } else {
+            // Print in either/both languages if we have values
+            if (localeVal != null) {
+                // Print the value for the default locale with it's own lang tag
+                writeFullElement(xmlw, name, localeVal, defaultLocale.getLanguage());
+            }
+            // Also print in the request lang (i.e. the metadata language for the dataset) if a value exists, print it with a lang tag
+            if (requestedLocaleVal != null) {
+                writeFullElement(xmlw, name, requestedLocaleVal, lang);
+            }
+        }
+    }
 
     private static void writeFullElement(XMLStreamWriter xmlw, String name, String value) throws XMLStreamException {
         writeFullElement(xmlw, name, value, null);

src/test/java/edu/harvard/iq/dataverse/export/DDIExporterTest.java

@@ -181,5 +181,7 @@ private static void mockDatasetFieldSvc() {
             t.setParentDatasetFieldType(compoundSingleType);
         }
         compoundSingleType.setChildDatasetFieldTypes(childTypes);
+
+        datasetFieldTypeSvc.setMetadataBlock("citation");
     }
 }

src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.json

@@ -17,6 +17,7 @@
     "metadataBlocks": {
       "citation": {
         "displayName": "Citation Metadata",
+        "name":"citation",
         "fields": [
           {
             "typeName": "title",
@@ -307,6 +308,7 @@
             },
             "geospatial": {
                 "displayName": "Geospatial",
+                "name":"geospatial",
                 "fields": [
                     {
                         "typeName": "geographicCoverage",

src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml

@@ -34,7 +34,7 @@
     </citation>
     <stdyInfo>
       <subject>
-        <keyword>Medicine, Health and Life Sciences</keyword>
+        <keyword xml:lang="en">Medicine, Health and Life Sciences</keyword>
         <keyword vocab="Keyword Vocabulary" vocabURI="http://www.keyword.com/one">Keyword Value 1</keyword>
         <keyword vocab="Keyword Vocabulary" vocabURI="http://www.keyword.com/one">Keyword Value Two</keyword>
         <topcClas vocab="TC Vocabulary" vocabURI="http://www.topicClass.com/one">TC Value 1</topcClas>

src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml

@@ -60,10 +60,10 @@
     </citation>
     <stdyInfo>
       <subject>
-        <keyword>Agricultural Sciences</keyword>
-        <keyword>Business and Management</keyword>
-        <keyword>Engineering</keyword>
-        <keyword>Law</keyword>
+        <keyword xml:lang="en">Agricultural Sciences</keyword>
+        <keyword xml:lang="en">Business and Management</keyword>
+        <keyword xml:lang="en">Engineering</keyword>
+        <keyword xml:lang="en">Law</keyword>
         <keyword vocab="KeywordVocabulary1" vocabURI="http://KeywordVocabularyURL1.org">KeywordTerm1</keyword>
         <keyword vocab="KeywordVocabulary2" vocabURI="http://KeywordVocabularyURL2.org">KeywordTerm2</keyword>
       </subject>

src/test/java/edu/harvard/iq/dataverse/feedback/FeedbackUtilTest.java

@@ -69,6 +69,7 @@ public static void setUpClass() throws IOException, JsonParseException, AddressE
         systemAddress = new InternetAddress(systemEmail, supportTeamName);
 
         datasetFieldTypeSvc = new MockDatasetFieldSvc();
+        datasetFieldTypeSvc.setMetadataBlock("citation");
         DatasetFieldType titleType = datasetFieldTypeSvc.add(new DatasetFieldType("title", DatasetFieldType.FieldType.TEXTBOX, false));
         DatasetFieldType authorType = datasetFieldTypeSvc.add(new DatasetFieldType("author", DatasetFieldType.FieldType.TEXT, true));
         Set<DatasetFieldType> authorChildTypes = new HashSet<>();

src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java

@@ -89,6 +89,7 @@ public static void tearDownClass() {
     @Before
     public void setUp() {
         datasetFieldTypeSvc = new MockDatasetFieldSvc();
+        datasetFieldTypeSvc.setMetadataBlock("citation");
 
         keywordType = datasetFieldTypeSvc.add(new DatasetFieldType("keyword", FieldType.TEXT, true));
         descriptionType = datasetFieldTypeSvc.add( new DatasetFieldType("description", FieldType.TEXTBOX, false) );

src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java

@@ -45,6 +45,7 @@ public class JsonPrinterTest {
     @Before
     public void setUp() {
         datasetFieldTypeSvc = new MockDatasetFieldSvc();
+        datasetFieldTypeSvc.setMetadataBlock("citation");
 
         DatasetFieldType titleType = datasetFieldTypeSvc.add(new DatasetFieldType("title", FieldType.TEXTBOX, false));
         DatasetFieldType authorType = datasetFieldTypeSvc.add(new DatasetFieldType("author", FieldType.TEXT, true));

src/test/resources/json/dataset-finch1.json

@@ -64,7 +64,8 @@
             "typeName": "subject"
           }
         ],
-        "displayName": "Citation Metadata"
+        "displayName": "Citation Metadata",
+        "name":"citation"
       }
     }
   }

src/test/resources/json/dataset-finch2.json

@@ -309,10 +309,12 @@
             ]
           }
         ],
-        "displayName": "Citation Metadata"
+        "displayName": "Citation Metadata",
+        "name": "citation"
       },
       "geospatial": {
         "displayName": "Geospatial Metadata",
+        "name": "geospatial",
         "fields": [
           {
             "typeName": "geographicCoverage",