From 1d5377a50ab0630621c0b5e33352cff27ca21d38 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 17:06:18 -0500 Subject: [PATCH 0001/1048] initial exclusions --- pom.xml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pom.xml b/pom.xml index de8ff91863a..83bf08ba32a 100644 --- a/pom.xml +++ b/pom.xml @@ -205,12 +205,24 @@ org.apache.abdera abdera-core 1.1.3 + + + org.apache.geronimo.specs + geronimo-stax-api_1.0_spec + + org.apache.abdera abdera-parser 1.1.3 + + + org.apache.geronimo.specs + geronimo-stax-api_1.0_spec + + @@ -498,6 +510,18 @@ com.lyncode xoai-common 4.1.0-header-patch + + + javax.xml.stream + stax-api + + + + + stax + stax-api + + com.lyncode From 990d4eaa7784571ccee28a11abdc9a1ab1899041 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Mar 2021 15:47:03 -0500 Subject: [PATCH 0002/1048] remove duplicate tags --- pom.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pom.xml b/pom.xml index 83bf08ba32a..753565db132 100644 --- a/pom.xml +++ b/pom.xml @@ -515,8 +515,6 @@ javax.xml.stream stax-api - - stax stax-api From 15f29a7dadce73b76a08fc8d86d742327b1a22f9 Mon Sep 17 00:00:00 2001 From: sirine rekik Date: Thu, 19 Jan 2023 11:00:23 +0100 Subject: [PATCH 0003/1048] add custom license for France to the Dataverse Doc --- doc/sphinx-guides/source/installation/config.rst | 3 +++ scripts/api/data/licenses/license_etalab-2.0.json | 7 +++++++ 2 files changed, 10 insertions(+) create mode 100644 scripts/api/data/licenses/license_etalab-2.0.json diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 9b570e59e52..01393d9c4c4 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1082,6 +1082,9 @@ Adding Custom Licenses If you are interested in adding a custom license, you will need to create your own JSON file as explained in see :ref:`standardizing-custom-licenses`. +Other licenses used worldwide: +- :download:`license_etalab-2.0.json <../../../../scripts/api/data/licenses/license_etalab-2.0.json>` used in France (Etalab Open License 2.0, CC-BY 2.0 compliant). + Removing Licenses +++++++++++++++++ diff --git a/scripts/api/data/licenses/license_etalab-2.0.json b/scripts/api/data/licenses/license_etalab-2.0.json new file mode 100644 index 00000000000..e88709124c1 --- /dev/null +++ b/scripts/api/data/licenses/license_etalab-2.0.json @@ -0,0 +1,7 @@ +{ + "name": "etalab 2.0", + "uri": "https://spdx.org/licenses/etalab-2.0.html", + "shortDescription": "Etalab Open License 2.0", + "iconUrl": "https://upload.wikimedia.org/wikipedia/commons/thumb/1/18/Logo-licence-ouverte2.svg/25px-Logo-licence-ouverte2.svg.png", + "active": true +} From c8a59666c9fcabd4f8390b58ebc7d76d4c75a40b Mon Sep 17 00:00:00 2001 From: sirine rekik Date: Thu, 19 Jan 2023 11:44:39 +0100 Subject: [PATCH 0004/1048] add a line break --- doc/sphinx-guides/source/installation/config.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 01393d9c4c4..b72bd891ae8 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1083,6 +1083,7 @@ Adding Custom Licenses If you are interested in adding a custom license, you will need to create your own JSON file as explained in see :ref:`standardizing-custom-licenses`. Other licenses used worldwide: + - :download:`license_etalab-2.0.json <../../../../scripts/api/data/licenses/license_etalab-2.0.json>` used in France (Etalab Open License 2.0, CC-BY 2.0 compliant). Removing Licenses From 858f865ebffa67383b38873400bf149756acb123 Mon Sep 17 00:00:00 2001 From: sirineREKIK Date: Thu, 9 Feb 2023 15:55:34 +0100 Subject: [PATCH 0005/1048] delete license file and update path in doc file to consider PR 9262 --- doc/sphinx-guides/source/installation/config.rst | 2 +- scripts/api/data/licenses/license_etalab-2.0.json | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 scripts/api/data/licenses/license_etalab-2.0.json diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index b72bd891ae8..1871faba050 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1084,7 +1084,7 @@ If you are interested in adding a custom license, you will need to create your o Other licenses used worldwide: -- :download:`license_etalab-2.0.json <../../../../scripts/api/data/licenses/license_etalab-2.0.json>` used in France (Etalab Open License 2.0, CC-BY 2.0 compliant). +- :download:`license_etalab-2.0.json <../../../../scripts/api/data/licenses/licenseetalab-2.0.json>` used in France (Etalab Open License 2.0, CC-BY 2.0 compliant). Removing Licenses +++++++++++++++++ diff --git a/scripts/api/data/licenses/license_etalab-2.0.json b/scripts/api/data/licenses/license_etalab-2.0.json deleted file mode 100644 index e88709124c1..00000000000 --- a/scripts/api/data/licenses/license_etalab-2.0.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "etalab 2.0", - "uri": "https://spdx.org/licenses/etalab-2.0.html", - "shortDescription": "Etalab Open License 2.0", - "iconUrl": "https://upload.wikimedia.org/wikipedia/commons/thumb/1/18/Logo-licence-ouverte2.svg/25px-Logo-licence-ouverte2.svg.png", - "active": true -} From 82d641cb88f57ca72fe26047721e9b391027cc53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Wed, 10 Jan 2024 17:35:30 +0100 Subject: [PATCH 0006/1048] Migrate HttpSolrClient to Http2SolrClient with refactoring in order to use one service to query solr, and another service to manage index --- .../edu/harvard/iq/dataverse/DatasetPage.java | 4 +- .../search/AbstractSolrClientService.java | 46 +++++++++ .../iq/dataverse/search/IndexServiceBean.java | 93 ++++++++----------- .../search/SolrClientIndexService.java | 46 +++++++++ .../dataverse/search/SolrClientService.java | 36 +------ .../search/SolrIndexServiceBean.java | 4 +- .../search/IndexServiceBeanTest.java | 59 +++++------- .../search/SolrClientIndexServiceTest.java | 42 +++++++++ .../search/SolrClientServiceTest.java | 6 +- 9 files changed, 208 insertions(+), 128 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/search/AbstractSolrClientService.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java create mode 100644 src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index b79f387f20b..b720b6a1b5c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -154,7 +154,7 @@ import edu.harvard.iq.dataverse.util.FileMetadataUtil; import java.util.Comparator; import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.BaseHttpSolrClient.RemoteSolrException; import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; @@ -991,7 +991,7 @@ public Set getFileIdsInVersionFromSolr(Long datasetVersionId, String patte try { queryResponse = solrClientService.getSolrClient().query(solrQuery); - } catch (HttpSolrClient.RemoteSolrException ex) { + } catch (RemoteSolrException ex) { logger.fine("Remote Solr Exception: " + ex.getLocalizedMessage()); String msg = ex.getLocalizedMessage(); if (msg.contains(SearchFields.FILE_DELETED)) { diff --git a/src/main/java/edu/harvard/iq/dataverse/search/AbstractSolrClientService.java b/src/main/java/edu/harvard/iq/dataverse/search/AbstractSolrClientService.java new file mode 100644 index 00000000000..f36c4a9e591 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/search/AbstractSolrClientService.java @@ -0,0 +1,46 @@ +package edu.harvard.iq.dataverse.search; + +import java.io.IOException; +import java.util.logging.Logger; + +import org.apache.solr.client.solrj.SolrClient; + +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.SystemConfig; +import jakarta.ejb.EJB; + +public abstract class AbstractSolrClientService { + private static final Logger logger = Logger.getLogger(AbstractSolrClientService.class.getCanonicalName()); + + @EJB + SystemConfig systemConfig; + + public abstract void init(); + public abstract void close(); + public abstract SolrClient getSolrClient(); + public abstract void setSolrClient(SolrClient solrClient); + + public void close(SolrClient solrClient) { + if (solrClient != null) { + try { + solrClient.close(); + } catch (IOException e) { + logger.warning("Solr closing error: " + e); + } + solrClient = null; + } + } + + public void reInitialize() { + close(); + init(); + } + + public String getSolrUrl() { + // Get from MPCONFIG. Might be configured by a sysadmin or simply return the + // default shipped with resources/META-INF/microprofile-config.properties. + final String protocol = JvmSettings.SOLR_PROT.lookup(); + final String path = JvmSettings.SOLR_PATH.lookup(); + return protocol + "://" + this.systemConfig.getSolrHostColonPort() + path; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 9e73c38a5d0..189cfb5de6c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1,6 +1,28 @@ package edu.harvard.iq.dataverse.search; -import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.ControlledVocabularyValue; +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DataFileServiceBean; +import edu.harvard.iq.dataverse.DataFileTag; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; +import edu.harvard.iq.dataverse.DatasetFieldConstant; +import edu.harvard.iq.dataverse.DatasetFieldServiceBean; +import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetFieldValueValidator; +import edu.harvard.iq.dataverse.DatasetLinkingServiceBean; +import edu.harvard.iq.dataverse.DatasetServiceBean; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseLinkingServiceBean; +import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.DvObjectServiceBean; +import edu.harvard.iq.dataverse.Embargo; +import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.PermissionServiceBean; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; import edu.harvard.iq.dataverse.batch.util.LoggingUtil; @@ -14,7 +36,6 @@ import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.SystemConfig; @@ -39,8 +60,6 @@ import java.util.function.Function; import java.util.logging.Logger; import java.util.stream.Collectors; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; import jakarta.ejb.AsyncResult; import jakarta.ejb.Asynchronous; import jakarta.ejb.EJB; @@ -55,11 +74,9 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery.SortClause; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.common.SolrDocument; @@ -109,16 +126,15 @@ public class IndexServiceBean { @EJB SettingsServiceBean settingsService; @EJB - SolrClientService solrClientService; + SolrClientService solrClientService; // only for query index on Solr + @EJB + SolrClientIndexService solrClientIndexService; // only for add, update, or remove index on Solr @EJB DataFileServiceBean dataFileService; @EJB VariableServiceBean variableService; - - @EJB - IndexBatchServiceBean indexBatchService; - + @EJB DatasetFieldServiceBean datasetFieldService; @@ -138,37 +154,10 @@ public class IndexServiceBean { private static final String IN_REVIEW_STRING = "In Review"; private static final String DEACCESSIONED_STRING = "Deaccessioned"; public static final String HARVESTED = "Harvested"; - private String rootDataverseName; private Dataverse rootDataverseCached; - SolrClient solrServer; private VariableMetadataUtil variableMetadataUtil; - @PostConstruct - public void init() { - // Get from MPCONFIG. Might be configured by a sysadmin or simply return the default shipped with - // resources/META-INF/microprofile-config.properties. - String protocol = JvmSettings.SOLR_PROT.lookup(); - String path = JvmSettings.SOLR_PATH.lookup(); - - String urlString = protocol + "://" + systemConfig.getSolrHostColonPort() + path; - solrServer = new HttpSolrClient.Builder(urlString).build(); - - rootDataverseName = findRootDataverseCached().getName(); - } - - @PreDestroy - public void close() { - if (solrServer != null) { - try { - solrServer.close(); - } catch (IOException e) { - logger.warning("Solr closing error: " + e); - } - solrServer = null; - } - } - @TransactionAttribute(REQUIRES_NEW) public Future indexDataverseInNewTransaction(Dataverse dataverse) throws SolrServerException, IOException{ return indexDataverse(dataverse, false); @@ -303,7 +292,7 @@ public Future indexDataverse(Dataverse dataverse, boolean processPaths) String status; try { if (dataverse.getId() != null) { - solrClientService.getSolrClient().add(docs); + solrClientIndexService.getSolrClient().add(docs); } else { logger.info("WARNING: indexing of a dataverse with no id attempted"); } @@ -313,7 +302,7 @@ public Future indexDataverse(Dataverse dataverse, boolean processPaths) return new AsyncResult<>(status); } try { - solrClientService.getSolrClient().commit(); + solrClientIndexService.getSolrClient().commit(); } catch (SolrServerException | IOException ex) { status = ex.toString(); logger.info(status); @@ -1447,8 +1436,8 @@ private String addOrUpdateDataset(IndexableDataset indexableDataset, Set d final SolrInputDocuments docs = toSolrDocs(indexableDataset, datafilesInDraftVersion); try { - solrClientService.getSolrClient().add(docs.getDocuments()); - solrClientService.getSolrClient().commit(); + solrClientIndexService.getSolrClient().add(docs.getDocuments()); + solrClientIndexService.getSolrClient().commit(); } catch (SolrServerException | IOException ex) { if (ex.getCause() instanceof SolrServerException) { throw new SolrServerException(ex); @@ -1689,8 +1678,8 @@ private void updatePathForExistingSolrDocs(DvObject object) throws SolrServerExc sid.removeField(SearchFields.SUBTREE); sid.addField(SearchFields.SUBTREE, paths); - UpdateResponse addResponse = solrClientService.getSolrClient().add(sid); - UpdateResponse commitResponse = solrClientService.getSolrClient().commit(); + UpdateResponse addResponse = solrClientIndexService.getSolrClient().add(sid); + UpdateResponse commitResponse = solrClientIndexService.getSolrClient().commit(); if (object.isInstanceofDataset()) { for (DataFile df : dataset.getFiles()) { solrQuery.setQuery(SearchUtil.constructQuery(SearchFields.ENTITY_ID, df.getId().toString())); @@ -1703,8 +1692,8 @@ private void updatePathForExistingSolrDocs(DvObject object) throws SolrServerExc } sid.removeField(SearchFields.SUBTREE); sid.addField(SearchFields.SUBTREE, paths); - addResponse = solrClientService.getSolrClient().add(sid); - commitResponse = solrClientService.getSolrClient().commit(); + addResponse = solrClientIndexService.getSolrClient().add(sid); + commitResponse = solrClientIndexService.getSolrClient().commit(); } } } @@ -1746,12 +1735,12 @@ public String delete(Dataverse doomed) { logger.fine("deleting Solr document for dataverse " + doomed.getId()); UpdateResponse updateResponse; try { - updateResponse = solrClientService.getSolrClient().deleteById(solrDocIdentifierDataverse + doomed.getId()); + updateResponse = solrClientIndexService.getSolrClient().deleteById(solrDocIdentifierDataverse + doomed.getId()); } catch (SolrServerException | IOException ex) { return ex.toString(); } try { - solrClientService.getSolrClient().commit(); + solrClientIndexService.getSolrClient().commit(); } catch (SolrServerException | IOException ex) { return ex.toString(); } @@ -1771,12 +1760,12 @@ public String removeSolrDocFromIndex(String doomed) { logger.fine("deleting Solr document: " + doomed); UpdateResponse updateResponse; try { - updateResponse = solrClientService.getSolrClient().deleteById(doomed); + updateResponse = solrClientIndexService.getSolrClient().deleteById(doomed); } catch (SolrServerException | IOException ex) { return ex.toString(); } try { - solrClientService.getSolrClient().commit(); + solrClientIndexService.getSolrClient().commit(); } catch (SolrServerException | IOException ex) { return ex.toString(); } @@ -1968,7 +1957,7 @@ public List findPermissionsInSolrOnly() throws SearchException { boolean done = false; while (!done) { q.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); - QueryResponse rsp = solrServer.query(q); + QueryResponse rsp = solrClientService.getSolrClient().query(q); String nextCursorMark = rsp.getNextCursorMark(); SolrDocumentList list = rsp.getResults(); for (SolrDocument doc: list) { @@ -2003,7 +1992,7 @@ private List findDvObjectInSolrOnly(String type) throws SearchException solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); QueryResponse rsp = null; try { - rsp = solrServer.query(solrQuery); + rsp = solrClientService.getSolrClient().query(solrQuery); } catch (SolrServerException | IOException ex) { throw new SearchException("Error searching Solr type: " + type, ex); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java new file mode 100644 index 00000000000..6a98c7cd4a0 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java @@ -0,0 +1,46 @@ +package edu.harvard.iq.dataverse.search; + +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient; +import org.apache.solr.client.solrj.impl.Http2SolrClient; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import jakarta.ejb.Singleton; +import jakarta.inject.Named; +import java.util.logging.Logger; + +/** + * Solr client to provide insert/update/delete operations. + * Don't use this service with queries to Solr, use SolrClientService instend. + */ +@Named +@Singleton +public class SolrClientIndexService extends AbstractSolrClientService { + private static final Logger logger = Logger.getLogger(SolrClientIndexService.class.getCanonicalName()); + + private SolrClient solrClient; + + @PostConstruct + public void init() { + solrClient = new ConcurrentUpdateHttp2SolrClient.Builder( + getSolrUrl(), new Http2SolrClient.Builder().build()).build(); + } + + @PreDestroy + public void close() { + close(solrClient); + } + + public SolrClient getSolrClient() { + // Should never happen - but? + if (solrClient == null) { + init(); + } + return solrClient; + } + + public void setSolrClient(SolrClient solrClient) { + this.solrClient = solrClient; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java index b36130de7c8..60ab269455a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java @@ -5,24 +5,20 @@ */ package edu.harvard.iq.dataverse.search; -import edu.harvard.iq.dataverse.settings.JvmSettings; -import edu.harvard.iq.dataverse.util.SystemConfig; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.Http2SolrClient; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; -import jakarta.ejb.EJB; import jakarta.ejb.Singleton; import jakarta.inject.Named; -import java.io.IOException; import java.util.logging.Logger; /** * * @author landreev * - * This singleton is dedicated to initializing the HttpSolrClient used by the + * This singleton is dedicated to initializing the Http2SolrClient used by the * application to talk to the search engine, and serving it to all the other * classes that need it. * This ensures that we are using one client only - as recommended by the @@ -30,36 +26,19 @@ */ @Named @Singleton -public class SolrClientService { +public class SolrClientService extends AbstractSolrClientService { private static final Logger logger = Logger.getLogger(SolrClientService.class.getCanonicalName()); - @EJB - SystemConfig systemConfig; - private SolrClient solrClient; @PostConstruct public void init() { - // Get from MPCONFIG. Might be configured by a sysadmin or simply return the default shipped with - // resources/META-INF/microprofile-config.properties. - String protocol = JvmSettings.SOLR_PROT.lookup(); - String path = JvmSettings.SOLR_PATH.lookup(); - - String urlString = protocol + "://" + systemConfig.getSolrHostColonPort() + path; - solrClient = new HttpSolrClient.Builder(urlString).build(); + solrClient = new Http2SolrClient.Builder(getSolrUrl()).build(); } @PreDestroy public void close() { - if (solrClient != null) { - try { - solrClient.close(); - } catch (IOException e) { - logger.warning("Solr closing error: " + e); - } - - solrClient = null; - } + close(solrClient); } public SolrClient getSolrClient() { @@ -73,9 +52,4 @@ public SolrClient getSolrClient() { public void setSolrClient(SolrClient solrClient) { this.solrClient = solrClient; } - - public void reInitialize() { - close(); - init(); - } } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index 04021eb75b6..dfc1ced7c12 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -46,9 +46,7 @@ public class SolrIndexServiceBean { @EJB DataverseRoleServiceBean rolesSvc; @EJB - IndexServiceBean indexService; - @EJB - SolrClientService solrClientService; + SolrClientIndexService solrClientService; public static String numRowsClearedByClearAllIndexTimes = "numRowsClearedByClearAllIndexTimes"; public static String messageString = "message"; diff --git a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java index adf48e05f09..a9ef468d163 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java @@ -1,16 +1,26 @@ package edu.harvard.iq.dataverse.search; -import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.ControlledVocabularyValue; +import edu.harvard.iq.dataverse.DOIServiceBean; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; +import edu.harvard.iq.dataverse.DatasetFieldConstant; +import edu.harvard.iq.dataverse.DatasetFieldServiceBean; +import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetFieldValue; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.Dataverse.DataverseType; +import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.mocks.MocksFactory; -import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.SystemConfig; -import edu.harvard.iq.dataverse.util.testing.JvmSetting; import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.common.SolrInputDocument; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -21,11 +31,15 @@ import org.mockito.junit.jupiter.MockitoExtension; import java.io.IOException; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.Set; import java.util.logging.Logger; import java.util.stream.Collectors; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @LocalJvmSettings @@ -40,7 +54,7 @@ public class IndexServiceBeanTest { private SettingsServiceBean settingsService; @InjectMocks private SystemConfig systemConfig = new SystemConfig(); - + @BeforeEach public void setUp() { dataverse = MocksFactory.makeDataverse(); @@ -54,36 +68,6 @@ public void setUp() { Mockito.when(indexService.dataverseService.findRootDataverse()).thenReturn(dataverse); } - - @Test - public void testInitWithDefaults() { - // given - String url = "http://localhost:8983/solr/collection1"; - - // when - indexService.init(); - - // then - HttpSolrClient client = (HttpSolrClient) indexService.solrServer; - assertEquals(url, client.getBaseURL()); - } - - - @Test - @JvmSetting(key = JvmSettings.SOLR_HOST, value = "foobar") - @JvmSetting(key = JvmSettings.SOLR_PORT, value = "1234") - @JvmSetting(key = JvmSettings.SOLR_CORE, value = "test") - void testInitWithConfig() { - // given - String url = "http://foobar:1234/solr/test"; - - // when - indexService.init(); - - // then - HttpSolrClient client = (HttpSolrClient) indexService.solrServer; - assertEquals(url, client.getBaseURL()); - } @Test public void TestIndexing() throws SolrServerException, IOException { @@ -125,6 +109,7 @@ public void testValidateBoundingBox() throws SolrServerException, IOException { assertTrue(!doc.get().containsKey("geolocation")); assertTrue(!doc.get().containsKey("boundingBox")); } + private DatasetField constructBoundingBoxValue(String datasetFieldTypeName, String value) { DatasetField retVal = new DatasetField(); retVal.setDatasetFieldType(new DatasetFieldType(datasetFieldTypeName, DatasetFieldType.FieldType.TEXT, false)); diff --git a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java new file mode 100644 index 00000000000..d6459626faf --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java @@ -0,0 +1,42 @@ +package edu.harvard.iq.dataverse.search; + +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; + +import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +@LocalJvmSettings +@ExtendWith(MockitoExtension.class) +class SolrClientIndexServiceTest { + + @Mock + SettingsServiceBean settingsServiceBean; + @InjectMocks + SystemConfig systemConfig; + SolrClientIndexService clientService = new SolrClientIndexService(); + + @BeforeEach + void setUp() { + clientService.systemConfig = systemConfig; + } + + @Test + void testInitWithDefaults() { + // when + clientService.init(); + + // then + ConcurrentUpdateHttp2SolrClient client = (ConcurrentUpdateHttp2SolrClient) clientService.getSolrClient(); + assertNotNull(client); + } + +} \ No newline at end of file diff --git a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java index 72eafcd763c..ebb29e688bd 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java @@ -5,7 +5,7 @@ import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.testing.JvmSetting; import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; -import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -39,7 +39,7 @@ void testInitWithDefaults() { clientService.init(); // then - HttpSolrClient client = (HttpSolrClient) clientService.getSolrClient(); + Http2SolrClient client = (Http2SolrClient) clientService.getSolrClient(); assertEquals(url, client.getBaseURL()); } @@ -55,7 +55,7 @@ void testInitWithConfig() { clientService.init(); // then - HttpSolrClient client = (HttpSolrClient) clientService.getSolrClient(); + Http2SolrClient client = (Http2SolrClient) clientService.getSolrClient(); assertEquals(url, client.getBaseURL()); } } \ No newline at end of file From 53e925304b703a7728ef56d58b2756527f83f366 Mon Sep 17 00:00:00 2001 From: "Balazs E. Pataki" Date: Wed, 14 Feb 2024 07:57:57 +0100 Subject: [PATCH 0007/1048] Add cookie consent example --- .../source/installation/config.rst | 120 ++++++++++++++++++ .../img/cookie-consent-example.png | Bin 0 -> 102967 bytes 2 files changed, 120 insertions(+) create mode 100644 doc/sphinx-guides/source/installation/img/cookie-consent-example.png diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index a7d7905ca4a..cd60f637e0b 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1382,6 +1382,126 @@ For Google Analytics, the example script at :download:`analytics-code.html + + + +2. Add to ``analytics-code.html``: + +```` + +3. Go to https://playground.cookieconsent.orestbida.com/ to configure, download and copy contents of ``cookieconsent-config.js`` to ``analytics-code.html``. It should look something like this: + +.. code-block:: html + + + +After restarting or reloading Dataverse the cookie consent popup should appear. + +.. |cokkieconsent| image:: ./img/cookie-consent-example.png + :class: img-responsive + +If you change the cookie consent config in ``CookieConsent.run()`` and want to test you changes, you should remove the cookie called ``cc_cooke`` in your browser and reload the Dataverse page to have the popup appear again. To remove cookies use Application > Cookies in the Chrome/Edge dev tool, and Storage > Cookies in Firefox and Safari. + .. _license-config: Configuring Licenses diff --git a/doc/sphinx-guides/source/installation/img/cookie-consent-example.png b/doc/sphinx-guides/source/installation/img/cookie-consent-example.png new file mode 100644 index 0000000000000000000000000000000000000000..0dfe1fb113baacb4ec89e61a17e08c75c96273d0 GIT binary patch literal 102967 zcmeFZcRXC(8b69iv?w76f`}x7AbKZ3)Tq&W?&>silw+WM~fsvswUi%pJ=f`Wo8B`Kzaf^r7} zd}`eg_jB94z5F~)6*yJy1&|0a#MH`~M_oy%onmHz#gtdyB z2Q?NRX!ILKld|1+B>;v8nW#&d%F3cJ0QZ-_6_Co%v91Ne^+_`3W+`>XUF$d5b!xi19Xp}hSdDkTM6 zKNvZfnAn2M?Hn0o*zAA@SRW-dKqx3A&u_n|QcBPEfcD2Nl+_*8W#97|+1W4~7~2_| zFuT}%yln?Xz=aREwJ~utc;aGXZ42Ua5q$c)1RrpJ`?cp22sjv<@+pal|GPTypWst-N5_wREG%F!m>JB@Z0BIc!ph6b%kr9y zg^i5~D8U4BwRJRbVX_6${;QF{+7UAW897*dbhNOueRA8bfuWs~qu|r0w}k%l`By(p zTrB=e$rkkQVF3eVxqZXJ%KVz;KWzh51#X}6DOk9eSZjz`*Z@2OXbACgycYOf{{MLM zUy6U!RR6Ok$DcL-c=OMiDj*XFQ9B!eq@&P(r|aL9|9tuHiUKURBmaXF|6=p+rvRUY zumxEDGiO5B8N7MQz%)L#5R<>X0z!8CK^+JFy!_V{xJSiQ4IN7@LqU0iA|>`#*#&hw z`ECPo`+OkAMddrx2g;E8dX##thrtk(yTk;F_nY55c>7L7N$epjguht?Qm-Y}-f>CM za7dYC(mzM!zB>!5T|2l>y*m|`oa|}p?A%DylxGwYZ*;Tt1R(VW1&zQ51^wT*oC$9z zDfrQ?L*<@6#rWsTr#?QwzMST9}EA&^cf8)t*jh;^^XjQKwzeeTpdrC_QkQTCRi1-vYYjzOLG#vX;x+5)2`-UO9H(|%t2x4!ORK0 zIGukppT6WGze<{}*+_L=(l+4LcDP@MK0Uxi-}aDO>jZojy`M55?`Q(-L&R_CNoT<<#9lVQ2k9Hoexp*Thu>zi;Ag?;oBWW$NxZ!5~-i(n8yUZ(CL#kSGm9Y3M;buMU~@h38cX? zsVHb%p4qOVqi%}Hd`>-Efl+^(lQ(CCk2uV~ZOqhy&V=6JhVi0RkrSDfwv(_6U+(#7 zICMWR*e{;J{+kb$D~Hx}b&{xI;7{ClbB%1;Y}i4wXF<2-d;#i>VS+JL&nSh0msS5p zr}^^9M_?urz7o;(higr2x&h2k5HjhQ z;7xOOY9{97QT*!t2dIfexO9IGM_~)_!$;(m(5=tkg95(zz`h0${W({;PJoNOK}UZp zH@u=^&8uRqSbwgKQ9i)eK8kvXnkY1qMDtW1l)ixeCwDC=0g*qK3i6Rnwdf~!0O1|_ zi1xSPhuseU*}MNQ!>4WIGwo0E1|==M{*%NL1vuttUimlAOy0_!($%fQC;A0<9|M!p?Ziie-uok37nTf#_ z5TNt7VH=_yq8Cy{>P~V*lA|WxYfm!hBPu!gauGd^P26g#nEg5C%)p!k4;E z6mI%5Xo+wB)Q<|ZSjMD~AOkf6roVLWqZ|MP}(*^0s-ky%h-Wo~WSdd7Wj&f9w@Iyh6?= zW`qe0TUQJP{{3GW?3NG!6xr7!7B<){sV(#6MODBZOp z)ov*#)r*bvWHV}EWqWt_eQN+NdstTG;jQ}7NkAk#m|fhZa94f_O-ZptL=cZNR zEmB{#T4<}2;mOq#{3ep42|0K1zIxY80>5fsoFZO}PVUP|ujkTAtR8Do7%$fnYyYlS zj0HT~siV{oEOc=RV%Kk088Rs-93FHOF3_qN5VyeGt{m`EH2S>zaTY?8J;hxA(6iiN z0+(A&$5Xb$2|U9!WDWUwkF3*jQ?9mCn0{hH*LybP5idild(H=W&aJ|7=MGlqCn&xL zFK*S#6#*oV6~?0Ll<2VKJA2dY2g9KDAb6m3w@L@jHRRP83RuMs4yy?ImxxA?Qnyf_ zRNH&mhmg&;?&}MI>)%e`aS^?kF;va3 zj^r&ek|^(_s*im!j3ufP4slb3!YkheH?*EvXSZC$EbM=hef7m9P0mjIWq3S9hqtH}2UZhrL!L($wSy`ALwXzI|`ZR~_%;c;3?J^Zy z=KbS#Z1t6JT(0T1F!FdfxkuYVV0`fTk(^`G%@x9X)em3T5$bIP4{|Y=WKu-I5~L4p zEPK_Id|YHPnj2v<{c<5Xe(oqlIyU}<@@8jZ{=H}YHlQ>Z)%XC}zp7Sr&D>kA@Ca=g zZCDbq^}?A{T5%TyJ7KFwPf>dxd_8OL$JGPDIAv$+7?;IvI;p)A@NK+Q&I_R@nx<*~ zW#NWhWIlq5@dq*s(BpBf)xlORQ}nCBJIMGkH9f(lNBltE&K2;wvs1k=CmGmfnGGL_ zB)Majex9(Yo;T>2cPu|^Qxy=YUQZ8BfSyc`JO*#G@~+*B7HPJ`iup=p$1X2;RdI%# zukBSlb;%QPn@kI=apI=n$VK4OUE9o7CA(hifRk3X_#r0uRCTq!|h3tWg+AzE zI_Vm|G4uo`hcgGf3xpp=p?hWN)tv9ouQ7JP<81)v{6n^)1G#5D-YH)p(TaV^s|;NEBJlFGI>r*^Fu_k^UP3%wX2J+k=g<$A#Kkga`xs;8!Yvv>E=SX93kYM zn4wkaV1LxP)FgFxyL>>7Q)gh4%3hU+lD~r6Mst93Cex8;p)D%oS2PB-*WR_c@Jy~{ zBYta;o$%EWd$vq<`OJdw&E?2?6PPeWhPgI#o-@aJTVIpfI|vJeYB=-jhSS2CX}wMLao|k7i`0DMc+C`!;#skHK8y=K>S#SLl;cZJ{YIsi#h-{f zCfi6};6$>!@R-r#G5QmQEA&Vl_4ZJMHYp`cK$R%z3S&-My990cj@1S3@TVjg?DJ^F zh_5cSTvo`LhUMW=`^dq1_an9Sj7qMga(-l`1g zQ>&7Ohsk{BlS_SoT7JA^l`Q!cMzwYl5@sQ)8X}6yxG=hu1cGVVH6<^w!27A{5vIlK zU{`VcQJO$1+c__@;$P#Obg7Q%8xE^vQ_SGbli3@T38HPmBLU>&36uq70V6C5Tm5)4 zX?AG_noIn$~=hg70fB&@TBZiEXSe5)NNS5M%e2jTW}+}-X-v`wMWzakLB z8fU7Sei35WZ?iJ*a;G(nf5*@ms=oWMUiTp0EGgyqd@Wm+U8mj&42{zeyaX!im5eC8 z1bl!>;_vaeI%(84^=#Wm{kvCRQ%AfAQ47H8hc85ZUo3#-!d-jc<fFY5 zqh=}jAA30tJpJGJ-HD=;w>NSB_;M|~hbWrS!GAGzODEBBO=orbr)b%n=t^c!uOGfp z7blyeaJ_{1oi{EK{VumiM*6^UUL7EAY zN?jvOG*#&EW^K$mp3Y1C=OWHXKnN-lQF!?#SLdE5?>PV!;z7C}tK10*=v&+x-qMvxb$_5@M2%eu^VLjFr zW_2wiV{=sRI4tPw#djU$oI)=Q>m4gl8?ZKYwX0bOV7{-SP`NBi&K&EbV$^rY(npD1 zG*?j!=2PX-R+*~$Y13H z!rY>PI&<& zM{>X{Ojt?qM^!&-4?Tb&y%*4WwH6*w?t7waFz7CnuzBlEXLbA~!iaMDU> z$2v{Mi}fuy*OXCX=dURY=xG^DoA8WT7jY%myM(xNgrKdO7}_810F=CX0C@b*pi zc6ThuNcU{AU+m6#ggbP!TPDL|-+EG^H*29Q*_^-ThQY7Qwf%FPFW;bjTDWtPq9eXB zR%aI+8en2sld6~fOiQ!p&=+IdF7Ssg;aM=6qy=rP53V1}_?M{SXqT&!~@;P`+A=%bNQ*jhnR-=bbE#9CqXZi@C1FM&hA;k4TE| zeTtE&>^ZfqEXUQD9oQrhR83fkMN^rZvkgp6->cj|FJMtSoildP#{hB%{J*nJ_ffT; zXXC{96cIAG+iN$UY|eI)E5<*%TcBG~?$;v%bR@SW3m?jjmR_4SAaI z2@5`u13gC69pYuA_*(CN(uEd9>nQFI!-%ZP=>x5>d6}}aa1aSXglS&(xAHE z+`&&^)>%_g7}K}kVj^M-xL5z>2Dvu5q?t&n>%q#G>6+nn4XUb`am`77+&cJD)zMy> zB8w2>pPr-Jm+Wjo<@$l;*9X{FN zG!=?;0MCOEo-ngiPm=cKTv{#7B_UF=i-mwoN|(K~s<&I=>mR-Oxjk;Kkf4h)wn9a$ zg>Pf1YA4{1gmj!wp%=UL^RW8k!0&V^E|)_gYhB}a3aD1Ya-7+2<77R{s(D*}5RgIO z#Z@8xJ-08F?s7>p>5r{vcI4CquTdvY?A_Q@7x4&d<0@O`9>_s2xwVBm5ZRhu^a`8_ zwzbDqil@^qYen2_CsV$__W5lF`ieuWSvA4Jeo>{G9!WJRopFw6sqwL~Wy-WP&j(Mr ziCpqtb*zQvqfJ=H>LkT-pcjVJkraNR~!T(+}Bdxu92H;LP^vTj7+ z`aOkRMMSOBqQF*{B?L22&PuK{pm)_F7Vg??wEfKpA!j}@o-|fe0EflI5K0hAWjP+o zG;hXpOlUxZ=Zebvr^7n&;C88%W`h@utK33tPqbR49;F=QkSjH4dF(&e7M14m<6W|? znX`tDsm@vX>xB|1mL!5PwwszL%iQDPDofGir41wN0cyH#G;5$A#I9arT$z-R*2xpi% zuU+OBaqNcE%cBs7-iv5K-^2dYi61lt%Ir5-YWDUQiz-Ky{=W6kLcDye69mbE@-Nv*Q&M<2VI8iOrgGokKB~_cMBOV@?8Q! zofRR$tI$;kLWwjSMsEwuNf$)I>dPKWOXeJd+Okn{zRK}HkZLPvs;c~JiXr;;ncx6t zCy*Se`T?REIWLh;lovi5?W+oS?{)23MLEp_hh5;X1mUPBZ;hTEg`~}hV(3|Md>^WJ zDX9o>wW_I}xs*HwO(aO9yh28dsGNlzTMB+g&A%&}7jd3nzR zQd}*iIZ_$5$j`PSuSpVD{n4$pUq1^CMCPO=8HtBG)=v$ysbx*;rFI}LA3`jB_>c{U z$%^G315atq*myE8(>X1J?e}%(dem9RuZ-WU7w#TrW17n~tyZjp?EQ>o6F6A$HG3-@ z$mL2F^5)tlC{C9a?C4-C-pfPl3LYu0k(ZC~jKa`@Y#tIqq;`pL*o?UQkYscR+u!Sb zWa}>va=B()Msn43J4QM3?*hQ2QRd&Q1Gac9fFzN}IyRR2F({B^Dl-#xZs8b<|&orZpM%B{>Bjc>~O-@o=WKE!XSn7mF+Lf?FdMZMhmED` zHcXLf(6k2y)@Z5|os7F+xw_ddLI)45^wW=(yVs@c&;sP=Q|2y?yNktAU@@<9o6j;t8y=IG zIafe;#51VexA#L_p+JhZG5+J>sWgSXH2XZ`oRoHw+O+By5Y);7mqredqduY2J<;l< zgcDK? zz3^z5AKGg$G%j@Po1X4HAk*dC)^7b8RpUZVk01U>=v8+XnHL9A8KfYZ^SsRXA${yf zlEw-c!Eyh*=US~s2NpW^SOf8_kJFpA@kN1fNy-MSYI0Z-)0K3MfD}Su4e{Nw8LJx+ z?BCg70X3biu9I&1hO!q)Hx7JJPDyqEEJ6iGZ;3!I53s8_%67ZZh#sP9(E|%jWHJ(S zvgUc8_Ldz<5Nx=FHLqVY-mG{wtxp22Q z*ATm9+sE7J(HJT)N1FxG(OxdbPLmqyVll~sgg?J&7;=XiW##- zaM)?NvVKi;*|HHooPb98=DH(l8Q0wD70)J@);MNXqx}g)*Bhrn<=|TM!hXGfN1*)$lSyNh!9#@X zbFy_yT;-!qK=vy&o>KJyHq=kXP7GplI@=^gpK1i+F!yV5I~eXp*@Zq#zq7Q zv{5Jpl|H*{#TTwQHf^8aXR_xs9qqzKCUDu*SpYd?OL~oekZy6z$|1gNj3^&N;Yc{I$gXY~?$U+J*|t$On1B zomUS7t;{w*d7zyD*>8vFjhDaljfyxY90iSkrkR)jo@kY}MGIy{+O(1t?{PhEL9@(t z&qQQ}bq|qhgIf;OIla7qZN;lVZPsYZGivV<*s4=~QUthuk~7o{w1ZqCih_^y`MQb5}%#JSlQgRID6J2_@( z=;O>PEMW?4NPKw2J`m@=TRR|dHu_;M_(E|1v-9pO7QS4{L8irgnvqLaw0)f&yVdr4 z+k@}){pV*FhynLbCy!Gn(a`Y~8Rli7*Zi{iNE*O_2{^dFa5 z(TFQSoU5MmvclJaa;??3Nsd?~pzTPp_C@wtR&``PG-qpgef7J)Z6TF0ILoz9VwUqn zpo0tWujkMiSoDu)`ut!q=Rhi3`G86k*h@d0?|vo-Imn;;EV@nZX}_NdvW!7%bZmMA2&019IBChw4d!*}8)V-JFmZj&ulC`c-il+LsUL$yx zjiEv4^?o#DUryAwwd1n~V4Sz>Z%|h=!i7#^C7QhCtR0_JDJEN_h&407@c9=diAWy> zRo&;1a`6|e;47ZXD7c5m;lACN=$y=$tyF}gNj1e_RJ2UvhcOl4PXfBbxM&BYM1L(> zeG91mhWVKHvTjcyedn~U`8u6*LcKL;vd$Scg3n&)xaDE8dkk#X#Q7EZTdgE#%C#JI z*c*i|64tq-U=FNItF|V7_Mhvl8QC+%1u=KD#}4dI05Q!0X>~}U(SF(>!0Tcu#(BoV zXL7hyZbGx%iCaLny=s3+44kiU8SFYS{16hPt+^80!k4q=Dvm3V#$Pw4C@xd80PI&! z18As3FU_a=!KMJ_9~Jd{{iN{?$3tNIoaDY{hp+NbtbNpA7O>}U$%{zEyXrBroTDT4 zB52O*kgdM4iTek08S|$Njq73%MI8$&QpTwO8wO3nb2D{XUGh(wUGy-P3gW)g^ZIt> ze5Y_AdGnKV&|GVy@R<5aL5)&oGZI^!Q!W?v**5DjGB+f5w32*0zxJ0DdXifeCx@2R znI)cWsocG^Q9tBCZ^^?3$62Qd``+WW!0~gOXqNu+ts3b^qz;;gj=y}aO@R#Wz^018 z1^y%b4s_9|a<@ywj{te_)^I!Lr?xA=ywtsDOevhiqu`r7Iqy&Q){;ayzsQb2c&bZz zae8RA)|A&eJb(nblLOqXb>1UD>cQtMG(Tl${Q5t6Tlfgz%rkPkie8l&XZVhWf6`|w zih2+6f{d_`n0~CKh-@E~n*8sUIx-roo=lHcHxPL=u^5wQ(w&s<)(pK=xpc|{z!oQd z%^EVmh=W5z#?wX38o#hfuC;`v{7_ZT{kZZzO+8>i_}`We%(=${xr(Njs24!G{!vjf zMV&{0hKTocaTtd@gFZ1emO{~1UTv~n<5>)z3D0o{mB-%O>G{I+JbwCv_ii?;Yf0s zbsFa+|CHFv6OYtdS0nMTt2AjVs_OjEY9X3KQ(;w788bE9PqXc?x-m4kJ1AdMG<0Xl zwqa<-sC}EIsZVEuN1zG77E&C4r*hc)snKq_t_Ii4Udv8=5l8nz-r1F2faT|0iyRiu z`Fchl9^9N4p8(v0cvQ>X{61ih-9Mi9kdeDVh1&7!D;Rf@ zp6z&{Mlo1DUHfVF&mP>q{7?!r;$e$+ML{iczMY9jvW(jPXZxM0anxRyKRHUZ%dtu; zcB46x65>{+t#DnHTP0_SxA;^UiA%-p)6BvtR3e9%$q@OrbTk!{>A~^OW7GvIwSs2@ zHQ~dW1-b*|&eBm@kEB%2&AfO=)~}R#(+nD?0!{-ajN#)1`vXmdjw`XWoFSDb8trFQ<4H{-_+!PuV-!^pPA(yT}gZ`#M8{x~r zJ6}uMY5zj4t%&}xL{lC3`h3yFk`{nX9PHBSA1*Ec==E=3-b~n<#b0cDnAIAE^L8Xn zfOPGw(^2-&|BVy%#sA{TTqcM6v)8a!aQ{G!h;e%gK{_5G72%tWNOwp|s0e|$}LJK=!yL^2L4~1?n zaduB5Wtc1~xR?NJ#%~Cs@3hpdvaJL9VHo7YgiQ*J`@e8j2+Hp*z0*Ib{}+z?s`)qP zgl;(pl=!$n_9sja=k={rehv3w3Ze%}XuAab0q|@_dw)xI5s=pZSF&yI3tj+-C0@|A ztOvKi-@&s0+Gi7(0pXkmn#{-U=+=fmFelq`_x|GCzo8?*C22C;BqqFbE)D)1K-RWwiWf+JAEmIO6k`>P_>D0KhzjV2A&Swe$gw7NNTW z1Rnvh=l=-)SCjy$`X5XGCp7SH5$bA3^Awiryt+()dYJ@l$_G%_zhj|lDJ79|+W0yPAVJD92J*;FPF zJYT$3h(FMLo377$0i`u4N0_jq8}%fTU7!K&}O& z`%~p+iB)t?A19poDxnrthjP)_IWP1auW-2&+!5Y`@Ak`N&f_Y_wKHBw0NHO8G@O^d z141_~j=Tkd?&pW@C$AH28x+MB&q#GSVv*GAC*l0r(Y)(ESLq-NYhWZjZKW?d^4JCx zqX6h|b}An*fcK;VsO~M@lrVW9f1$}S-<~vmz1)_$7nkIt`3asp^Yilhg1R^hyf=0y zMDw%#m^^T}fpe8q$06QzjLWS%d3wQkBz=mR4E6ea%Wf4wM{(h>G)NY0`}Vs`GHf^J zVej><52x-_tJvY2|z2gah@$8)mW@_oN2sb4=Zh@7 zCbo#8ulD(UpwVTkTRbElU>iOFhDX`u^BXb%f*Qj$L}1!|+Cop3<8Y#)(^6Kq`@z;C zLcTc$UQRH70Guh66;1|>QT)G+QO`HP%v5|0crDXGD%Z^uU0$+D7z1>|K?Voix;9Jw ztdkVr0Xb0mHxGypdCdAB9|CaNM7zd#lQp+%+fB_31zA-+SRv%{fHn2d*$l{-UkY59 zedxT_GKmh@{Vdc$o_$)&*$kO;+lo`6;7$Sm^wz@GLnd%;!MhT+$z>ClE1sPK_{f32 z^Ajzw6Es?tEtrW5%sd=BF0yce@YYw=w0Ku8K9aq9bImd$owi!&b+D1MT2Ru&Rls3w ze#Mbjv7d!#x&fr66R|rNB)Z;s>5vVaWM-=GE9>uy>bIx~4-xXbm<8V=p{WLftT;!_7R-vfS|#xGV^v zw2HreigSr)^#_*;6rElHF=hWFUEs)9S)*On^1~-xwHpWJp*fELe1{QI?gi%ywfF-; zwb}yY^$vKh)YXgzDU`P@AITj=Ke0#+m***QT?Rj{8{HM6MEiDJuaSG%Hv%tM39C0xFh$IDermZ6)jJ- zamb1jb1ci{9{EZE*$MyQK3!@}nfI}Z8t1Wff?2QHg3~nf3;7=JMjjJJZl}q;~C_aYaHSn)h>*>8C7vVL(PJto!_DisOW!=_RcCk>?=Yj0?2(_(Y2NvC8gB zm{7h+hlir)j9tRV0@w8jiDbzWYolLwYu)?-CQ^WNiu&Gn3_%Hg<Kz9#X&c$yP6^7b!^FSW=QI~a!};v$SX$gw5|>6?1+n)$}lJ!e*M^t_He37hH>(D}>#PHM8)f+T^J zwklJ&O;tY+)$YSYIwzY(p7vA?*Zlx{?%IH(y+;0B2$c6{8B8T+%d>u=Be9P*TyU3M z?5Z*!XSepk_5?mY{NC%pg1zV9d%el!WKw9>^|Zq|a3U;K)5-eVp(SV|P%l{IVSawC zgO+#>gX$L*O#E8g&#s0CQ4ctZJo7kTkEL(E`z+MFmf{&hWEUY(|9~VRCw2U1Q1IK` z3IEEIW!RtYU!u`W`1?HHdVE@L<*LKVnGLlt6MOh7fL$uLZiq$ zuW*a?Rari}O^m|nbMVa&IBw*g>dy-+=O?T9q(A08AL3Wpea;nZF9mEotz-+{e8SNz zYG|4q_pmci!oZ*L)}u3ALS6$$Lt*W)$L!}S4T;kv1NWShN91krb4p$Zg)`jEcXW+) zZeQIluZwGpMS@!-3^j^+8{Zn90pRvK#p_o0aBKS;E`cSnAkv~*wAPsR#$q12MP}N| z#D!FRf5iCHYdi5_pnr&TJn3;tGU(dBnEL}V zHcnBKA@$V34qjBvHd~jpJFMn4yqftMhxfSdqyYkcD8FJB`t)>0^_M4=TD+(D+6WuN zK6Z8fihIKh)N3YG&f1*P8Y_xrXZ_wWq?bOlglw|ulYX9c?J8QOH^T*NZ=ztx-p6M9 z$RfW3LTr861e_m1F7^bL5Ad;Ts-5SVr8tQwXIOGPm68jI)wQ&}wlnWNJ}flHPL4TI zT1sdN7iXpq4_pyRTngvBK-7aBHCT~S^-1yv(6FZoCZf0%MjdmH1WUw8Ikjtj&vxq( zI!n>aid4Jz<{I zBi|*|Y$0VkQ&$4Ca}4m&T#`BuL&X)NCdLwn`8PTl$6adGhNtdu^PC~ZYiOU!K<@jL zQO|>R`d8co8*^jPO+kU1>NqSFkodDPHATNVw+|)j?)J*X^wx+)L}DXTFLSx$QDg3c z+WZNtQD?0T0VGBHG*}zQVziW>0PE}W3<dvtoyZPW1vZ2 zJdA1!+n9KdkY#~DO3yr~=5ucc0I9~S9w+DIkPr+zk1KFs$w_%QZwdZ&eNl_=3Y#Rv0RKY`*K8#~r()!fQACHpReqSuWdK0IhxX?5GyECF|1kA<&fVPp+i*M&QzRA7J8%M#k3rBSN*Lz=r? z?~Gu`>_890VPN1WYs@pTJFXcNSEF zn+~>CoGFz(`=E@^^vAYt`bO81kkFHv+3jki?$DqsCmR4KZDqz?F#wK9aLx7$%wp_# z1-t68F^BA|&kE#SS%g)N=c@=ITQ^lKhCE-qZ-D8HLAT;f6N{=tL;wI!MwBycXl*j- zQEgkp@i@F#ZEM?BC**U5ORvmu&n9-;}(Ag3S2bja8ti~OJwZPO-Bo-~7da5{ZXhm0$*j-@b4JHQ-e?O3xQN=6gRFt_ zN@ChpGzLoc#j7z9m@ttC4T@zH- zROrcGY^~Wic&|NH*ZDZuz^}yFaK=4XI;D>^2Qo9tzS}T@=bWi1L)+-!wo20)khrGj z(Y+IVW?Rn%hF=c=$H5ji-KXb0%4KB>Vgl>)-Y(|atwvRKYOkb|b9rX4p^cB=ct=># z;G$bg(jvIfxL8ttWeRQ5RSA!&d{+MYAy~VLH-V5v(^*um7t@><711K{f!gF)^)x6S znW2XjY%Azihm9;|N{A23NqUA1|ZY$R++D4}InnMGtj_dz{k7^58V4$3aXbpS?O z;ZGVkgO+N=M_#NB2fL{dM5xPeMq;Y_M;8(1ES&;MA&5V)p0 z2!}DobONZJqGOh(^MqbM!a+^!7rWnMN1v9QiCM}v{xA#gk^z3StwXVO#(h$CSHXpz z;Omap6^3|H#eLaFftd)IhiQ)~$mg79zYOhO)0R+Z8#HDwvRg;0t5r{D$>;ldn*j?eCvYAs+2AEeiG#Ti#R8 z$#5z8R4LJ_?3x&L-l>0*5%ii0fLgVQqy}HqHRKq?UQCM<*K+hAAu}p|>EtA1lB8f# z&Ha;X;$XrlBk}vea*WifRoVrc+@b^C4xMtvi^b9&!WW)65)V%$r9IqHy-b>9LRanF zV#U;ZhVj1pXWuIX6fZ0Ji-Z0CqbC6k4n~`!1rvyixt<8JCKh$v!n}g@Cc6-=k_sV7! zkxRD-Wb@?X4VNC0?E_{*@W8 ztqZI=YCm?W-b-$Zk)Rgt#E#fg?x?z2=k=ybaj@*L=YPgi7oH4`wC_hu>h^bqGZ|kk zgy_K!D9&7uO6l18iKjP&_sGFE;VGobdv2qpHZ!~zsBwqD2KE2N-dhF588z>|K>{H` z0)*fY+}+(RSb*T}GPpB%aCf)h?iQTj?k+>n!6CS_-+bTxSMAz$>fD~2!}S#N&dgff zy}F;@v(&iH&_d(fk6zo(I-zKjwH=vG&Xv*n-Y2p9?(rym?7Z(UmrEyi+xt*6!k;=W z4neptJ?V~{9^;2xNll}9?=yPkUfA#q#Zlp^FCyfJ`Y zS)uSrY_4XPBKx{hKM?FPu{4OSvQG579TN~k&qCD($L|4HJ#V(PSutl=Q4ou8F}P=r zN7tB+8W-9_xcW%k-y6;SPH`NG2kHqVhLJj_N$*>!zYSu&b$3JF`@cho!m7q^7;oBt zHuF_)DD$i@I{Dm953<@~veHWEJps*2=-utSS`FJN;i-la`hCQp(#zNJVa_7)`=N=nVBnuj3-;S8>BZuG5a!-_!!VZuzvkTB=>RO(6{7DxHqg z;iEiF7iV|HKIRHXVR!&UV5XBiM#K%C^tN)*9tzP&v9O@OF zm7?dt0ljPz-2CKbJC};@s3UM&Ls*0#WC)!xziSkW%6Dp2-|t}+cM12J=o<89Y@zSf z?mTBYaw}|Ww)vHs-ynNfmQHoqPvf-*Xy|Fo6B*Bhy8AmuHK8VVN|4+8I{>gB;l|!L z@Y>Ui+c3%&B2tiFt1k0N^%T%zC{l;PfQuI!QciG><(7sFs=Xa~ zY)s}Q>*}0JJRefSy&;-x%2Ql)JIU6xc({DI4NB3AKE)iMsgT}%I4>hT+rC@7f_lf| zb~U|azcBV>V*^&045`5h^G5scrCjytX9{vrw*ymkap>_>5sz!MwmUD>H61I%>)>vG zTT%S)2f9+jzMhdxcqirJ0qnwc%VM{2&oJ7)!3M$K_fa1-yc@h9gs!jZJ$jXb79O)W zpM!?%1YtlSlx-m1tT>_Q_s3UN>scG_K8g>W;^-r_hEWgVf7K}t56p`vYA^Mi?k4zk zFySkVcJ8&wsBzKddP-cNYC`s!vRmcnDWU1^|?#xhEfWl-o2 zcdX<+QF7ZPe@2^w9Dz5uES|7+BpIaoY^IT~u9SJ)KF0httU2MGEMFsOtKywa``GS?yXFA$f>j<^k=+FjgmX9F$0 z{G6n+v5^wqzXB8NjJO_8z5ollI>+~UIvd^WaoEDc3Gk_jHX1VV^e<^cI1jAvSr8S zIJMsX(K3XZlz?yFZ>kw$Hc&m?8Jq$dCegH+^D6UFaDq)D-VzHX_C7{pv+kpIJMi=I z&gZdM?umy=*eqQ@;s?+yG*uw_jj0(x5e;>%vOovAzoEgVT|P4UTAzL)AAO9 z&}ZV2!d%C?r_z>Lz;Qr+;tW(SZD0d zR&bA%5c^(~Ohew~@X)sZq_D3KHh)#fw#3@JJxF>JSO=0PgT$(FMKhxk9~SMi-vCLA z`YytcA#`5|tq#M=lWc7*c8dC2(~)Vc@MZ0eC`?+z6n~LR8*;+&@+rE|=LzUn<+w5= z*qf#Nn6jGhqE9Uo^#iF852&vB`PWNdQL20fEBaLa-O~56EEb&*M!qwPm8I~Qy(zG| zE>=d(R!)+$v7ftGTcq>{k6WY(e_~EtIAa6_mjUL1g4>z3z%_F`hPC2z z>*2>5EzDZ|*K*n&Lt?b?@j8@y+ve7{)~SkKUhki+AX`gHi>B^$jeaQGrJi)C{l9<$ zhJ3eb>(l$FUw-rVK(p$_d-hsJ_ip@rvYMwdeN_t=s81dqKVf!OXy?pdNAAay-Vpq+B(skVgFiFW76sV*P;v8E3^B9q4i~SB zKRXy-;IGv>Wv=lCwt7veK9fZR$b1;TDPGZ&YKx4bCv-b>|N1dvEkmnLSb*G7hE|zF zr2Di9WL8}k;dLC7wQ+>3I_#JW_OItrJvB|cXP|SQFE4$cSA8xUo9XG=EZs+0xnx5B zUf+Ol$=ZRo(C@mVOj~1H8rB-tl-aG79t{m!ivYXrWJh62>i6qRL@C4GMXAbt9vQtCv_w~x)L1D zB~sct9l&q)JhZ1B#;0nnUN`D6oY#O{GFPd+$U#eQ>UlbkV+_S;qgHM9Q>Bnk;^4u3 zcwd>-7NGeeZGWUGs%`~SE=jLxmhu4~rI&kWRUzL?QTdb>6=)O1n6vyR^9O{Waw*Dr z%tUF$_21jm&=BW{cF6?p{``Hz_b@lhSdzP_r%EmD=HtMpMamwW^q;*<<>5Egdabq4 z>J(wd$2U4R%FtY)X*FE9=E-imqcXVAbBH&gx=V!! z7kYLKW35M%`(Rf+-xh^nr&`f%soYn4)C%1fDt;`RRkc|Qg(%%RO5SV}vSETHYlQ5c znqHpd^ud(5gJnjfy`kVsKbw9^aM54|dZCVjQ)uGCgB=|@E+oFdnEUKoaCjY0(9zl7 z1M^{MH!3lYTXbWiMmodNern~uM6gT6epEDwti9O=cM_;x;->M575Q4f@Ddg&u7%F> z-1xK(s8x`?ws7*g^3L8T{jHy(*d%zU9&6eox3o=z$nY^Nm@T_Mg>UQLWdyRX*MM;=5WUwuNMo#yfnyU)pHo!Xv|4p|W@3p+>t)sTagPHqcg3>7$h}qc!YY`NwLP-j!P?KHD#S z3a*?)HlP|ewe~c_yimWAq^wbVw+BEfes&X&#|bn#VJ~8O?x2UHk)iVS=^6?VEj&FB z+sbz)hY(v`31!U2s-f5<@pT1XcL!gp_Y%F_Ia!pVZj=$9`OmYyIZ?1Z3oZlQCo2>; z#pn~AfF=^a*7IVePy36U6N_;86Fh`o<8PV=>^>RPXb<1I(lg& zPL#ln1LdC78O2uO;8JdV4gyzk0q+U^E#m3ZH{jD@TJd>Ow%KrQ9>9 z!)iJBf8>+rjcNMcDGPRi)NBjrVXF?$M5PqIZWBS#f`cgyS;b}jvMJZ8~ByPuRnFwzcap6 zUf#evGVm0GuG%~3)syZWlTQ4{Z=)mHI3}RU_X!;+g*k5bV7`tVovv`tVz$;6ic8Kq zCnL`+->S<8J*@w|9Ci-<8%5`mA{P7-@Cohj3%yx`g^pmZbkvs%qn@<(c_q;*sQU;7 zYDm!k&&J66eAKK7dxRkNf-}c9#zWe2c4V)ybz~uNs+_3>6r`){_|NHVoli5)Cq7l$ zXm-ZhKW`gAcx6U}ZcSLz8X$MGX^GD+m&s494gZbwhaM*4Qcl|2N1I=lEbhu}a$O`T zB$sjvvws7#XT!y7f?azHR12?M&#bA|P~FzaEVk9E@xQJeg92H;&$SCN>*v$J+`jh6 zN>9)5nQQ!h4)GchBt3XvZ3wI8lvU9Wu~71;T&2nv&#mvkkIo668SZ~u5v}We@-4qb z)6v|ScmB9x`^J{yI)49)PV`C$zS}kn%LxWni0n zdU_T$=XpOrMi&pcA-DpM%l1=H&Xso<<52DN;)MRI*8uWtiJ_QNQ>F90*C%k7QTIF` zdxg^BSOPssteXf|+xB1`tJA`fV53Yf*JMz33avEj9T}XW<2-|bi=xo6%2~<6<@`#o zL-DosWV0nrFT3YyXNL9}>y?VCf}uuBlPSBr*zGQRj)LUYmO}8IiJwH(WhvnrzlTk+ zQYRISn@hD0Y}0>_kYJy-wUFA5_Nv@Nj5f;py`aG?w5NYE9mEg z7x+?hCngWPUGaGk<{ zmHS?}XA6%{Upna5-p)zOW0k^j>Fp8VZ^d~lc>dF4XL@kmLo-O=Ads_ltc4t_H2sS~ z|M_T|J}&OTlFQty1Az`MWM$rq&84>D{EbH3G;Ey7bocYZR;P`We+K*<7p{iTDeCOp2d-6CsZUP zg%zF-pyHu)IZ>THnVtx{rn~;~oRBFF@X*S(w<}Q-VZ4=^hM@;wdskogUZd?Qo@X{( zPP0YT&V95O%?S&8*Q1R-uH2%WN)=1lVS)5&b8P@17SQWu&2VsQViV$`1+0l6eYi4Z z#wN(($;PTkv7-ydDr1jZu)}|;KIMKr>qX@mQl;I|CDpo}4nxzZX0U-HuC=Lsr|H;u=X;hcsoHMvYP(ZnBhH z3cLy=b04k`Tkal&WO$ZyYdFU?X$$>C zCqy?{TC5xeOFJ8ajF&oXuQvamaE&-jVNkABr3$pm-DwC-e@bl-bU;H0y!@6OBgV(y z0}gkP=H0C;Y`waHg}u(nzXSBTOi+dbx-;1d>#h?{TOR}NaOs7hd9rFpsia2$T$@VT z_=-6$r5=+sLRgqUkA?Ot&n`%E0I1NT9eqw!psz;^Oh?+Ta^+-|(yJp;(s8v_iyL0E zCDWd{L5E9Te7A%#A2h+fJY66%P*z?37ngmJcvywfPu40yO6 zXX<*fV#kxbj!wd#b0ZVa&Ug_`ym^3JHIQ;;Qe`R9KS=Td(tG0=&bH%*d`+MG7PSxx zEvaSbA~hxt?vb*0cU##G&{h1ZY)gxJ51?q}i#i9`ybg1Gc zmgte=E{3UZ0Xs_27Sgq`h$!~T;nmNFJFJvTd_zl7Dm@PmQ~o|0{Eh-+hh1EC@^f9( zZT15+`{gNO!XhFWH%?{uXWX-&?`v;N)uP&91HG&{o$d8+l&qPu7~M|Eg{Y1Mm1;G=>-8=!C3jPVGF2ftKe9 zFZSf57Hr8;TGn`xT0>w}tX~p$q5}a#+;-L$b@526+p0}lV-#rc{nixJclX3=RN>y< z@%bxq83qO27Nm6Aa=0+rIdAkb{+GaRpQ~CR!)XWBrjS!`xuRbU(8{1bFmJfnJ6=SGXjjv*#{+NZmuu{C+%S&}=GtAchU z;c6m+W6PgKUi5Iq%VZ^tTCE7woYc@>ZU7g8bluhY*j^>y=1(ir3hlCENcSu-rC?&$ zT#!xsPIz?a80X#Wit3Gw7HhzHTD^$}4il<8<&s}P89f|~)S2f*E9%>sk~u*r7`)kv z^kuh7lSN8A5Sq-cms|A)oj*Db8#4sIUVR|4SK7<`X-o<)0yI}PG(dKg!HsRZJoStO z&vdB3#p1!R&&vg$KgGMe(@>;-lBfz|2Y}|N_SJtjt;fT^{E~df*;;0ARy4$raE)Yt z$b8m96SRM=5JO7%oNk!Ot|Qw~NgF9v90MV750q9LKs#*bzzkh3M^y$3`fj?jMc1`ew_jGTt)7Vmts><| zn{{^A9+h^rd*f}$%1*M5H&GYA!<3KYfKlh)S3v*T0xlUYqEcAnFL2cpbOtU_7eBPl zId*8QLzVhfdpqaEX6$#9yw9iYNMPtqi+`969pho{ zzkw=Oq{841`=Ritu;sfZj(5=LaVD7Zj8Sk2$0y@#kDHc#9v2T5hSV8@~6(1g~02n^lqA%bckkIzoipa`idhpmYF|lgkq~Qgqs2 zS|=?+kHFgCdQ#7~fJao1`u3PRpPa_l1|L#Q$L-pB(QvK;vTN-P*A6Y6!Th(93#E?5 z*`ltGgPQcjzhJ|y=_+?-KWO2xeh zl^Hu(449+s3dui!YXc-2tVPmGTieN31{xwU5U4!=*WxHKcKd>XG2FuNUDauK0ZzNI zc)VS!gS_6%`zW}>>MkKrr3%e+FRA~`vShsZM(@F(^m+y@g+Z|3-d6%pg|)t~5e)W` zR|3=twPo#L`C81^({hLG`oO?>s_w6oe}Y~)NmF`TF=&2O!&U)yAM}XaweeeBFEYkq`4}lX;+9vcNGrqO1hsPLM*JN&7oVasb|pBPqUBckeuY{y+NRF zKNoeS74D5REa(r$dRTOlnJZt+t2-;cd$PE!n_+>BqC-GFwh{&71GC1fA#_MLyT{C1cLayl zFTKV63LTa&ycT)4Hr0ni&Lut2^e8Xo^Bbf!ZBbMTI54XTDqZ00_@*+o9jV6!laWa;POvTdSLDc*PI8JhKDkHlUUO)Ewp>hHE9 z-kLATMF_V*;~Kctxp!;^UO%@T4LAz=WY%3DdbK{-_NTmx1qx@Ou-+{X;KG4a}q>x8mZpFTW>W~ zv<{`SK{xu`=-guJ%_li+6c7Ip<}BN$sPqCA$^w{HFkj+KlDTFqNcpET;vdp<}_svn-`dvgn6O%Z{+Uuw0wJLf01CsDG-K`s-@C1OG`Cpx zh-2$qn(<}K@@B@PUjZb?R%E3Vb@LnHEwyTT2gviUCr}Q&)we55 z8k+)D)6Q3Z-nTDLmvYYm>D?$*gWD4LQUMO8_gW6M6YMul9h3fNCS02lM*Hv_1S>`y zqVM}W{sdSDsRUz!MOoVy2bSPcweEJt~(ntkfG5@w{PFdEJov6?$(n14bVKf(L)@wH*1s6zN>PR z|5gFMaEzFd2V6ee_J1YA75Vg#Yr#pB&~j+k>ns6QXEqWu5OmfYhcgf}aY z*#d?fr)mI~A~^lT;|K}z9s@iW%>k!-LMMl_4||H_$GWrBa35IvH`g<<_rC0X2>GPOzvo1I zn$v+*9Vcs_!Vnwv+UDaZ#onfqI<*#{oJ}Zs?|MI{7 z{9rKfZT=h}^t5lx23lw~V*#DG>>{9y0i1}5hAXnt-XrzU{seG3-EOZoVfe(qrbJTw;a31S%7dN<83!|=M22~tugs1@!$3JYuTejV4Z;XYlGt*M=We3yz!0lS~ zHK(KAZ!vHG2XyoR=651B{f|N$ zEfcJhko6AF9CIf77NcXxdtZ7-QQ6XZ?U#)A_4T6Fb1({C zF&rAVFrm*p^8~J7DL(b^-Q9A zyBg`AmdDfIL~k^U>m6n(39F8OBZhQ-INIxcF)P$+u}aDtyU5Dx{EJ7UsCiNCUc0p- zgNkIp`wCp9U}ZNPLZqd6(zW5kt9u#ZB32;cL95oD%Bu{?x)yWT0d7E1*MH{DFdRXGxIf!utanA@aw%2b zUC^zoH(O251u&M$U%?I|oprZ;QF>-NAtTz1dpexmPXBoMlBoYMDR8qU*45VKbrkt> zNo8FjiksW@=&&u>e|hEdc$(3g@q;kNdm{D6-x)NaiX@l4uCjPm#-At`&G$EKh8vlJ z=F8Ag7os0>4zmYbC(cfMlcP&9f+^=Y-Zw4j?$GaqF0+y-b4pT#xs#=rO4lUK?VolY zqq`|zuL=C4qAWMbYuKX!ZR^kU*R{)5!@AA9%L>(PRF`mfziU~IVqtEC;>uOGnHcAz z8Ko@et*&d2XUTDX)?B8$mj=+fXco;;V>||L8W7CI1zvV$!+)nS#O6v~AC=6$dZv5x z?0U6)oWylHRUS?D&UPf^di|%lFVk;!-C1C)?sy4K_nh<&kDf`!GzI0hHiCx!Y@S7S ze58s)fTLMRX9Pn91b)_DzE^5JO)sCvO3^japSZlItL~^eZF?oLn=&Kq7Nh;E?|U-8 zr#wE-xcFh=4o z-m%#Vnycj6C_(%=rlFX_Bc&La`fEw=FmO9YC2OVmFQvzrfRo8oK=37?1DX~e*MSb1 zOaiI8`m_YRib@l5Fb8!$`u-{+Hot`?_q@GA)r;jxzFNy7=vVJh8V~2i&nKQgUjB*D@TeW!2x2?L~0=f{Fp^>4gDD~Qo`G4 z1%E@V@Q>j-fY8sP~5?P8eE`!syWH_#6!o;e)iEwtM?r!Dtz~2}W7&pm zn@?FkI-(SC{7+^&)Lqwv?50t(LnB~zZ7l?27# z4I>=Xq}6ho^b6H653{~Goyku@7OZm+p6#bUDVs0vP>6wXdTgqMZ3Gj)GsOK8V9WLe zUaA3)&U7KQ0hfRiv?3isGVa_oEklaJo(C7?C4zx2jg_8qE!rBDB2snfogwQpyFBZ* zsU0yFI&CblmRg8_#+I-6)Cf{b`8~LsStRX)q0z~*<41O;+A>2@7MYk(ZK_5zPADq4 zI?@-u;h#wIrg_~l;X!(61OxXezZRlfffPD9bkYo|h~~S&nQz+=eqNu}J7(x}x<2F} zB3cVcj7w%n?C=Q)F2Q1*wF_jG*0VXLRWBol%Zc_U7Yt)Hnzf*<{w#>{q&i-M3|qM!@5sHDM53K$_M>7AndLUFsH&)5FfbC+Y|Q=Wg*W2MB| z{<$fdptKEoh;vc2eVjidIj6Lq`++Hk>53eaV(i5G-O;0_-W~tnjD1iHl3pK{`(O71 zNAsRDK51u@ZBv6|Sv=#f(tpmVG0bNSe!Qg;maCZ>NJ?q#vOVFo`zGH3TS9 zwUF6%ZP&%t{S+-E1H0O!`hf##_o>(V_w5B0w9eh2VuHK{UTcYw#%XWwX*1u9U zU+b%oX^bi!S4nZU?_UZVBysnxrSokBJN_OH%9H9AS}Jri<+qh&eT&j@q>tgM$qgbi z)lS5Q%1xkRRPfeN;I;+k(r0=6a@=~T`+;(2n#JKlOTeu(<#Q3XOrH#|u~0TZHQsVS zEUM5SoQy^6AuZ-~4}Cq<|I*96AwqQbKo3?4Yr+pn0~^c=VC#Lw42`F_T}1htB6>09 zXiyZ;IIO?GM{fTpfjM zX3GBKw-=B?PrG%Noa)O5O$lE>et7t^=OU?4ldc7`V6`@K|F_=ad7)%C$LSRG6hmdeELz_ox+ z(n1Sm9+)mw7ScAqby~zyPI(0oT%+)1)`e~~bZ;An>r-JA(^=&cuto-AVZ2o-(zt8; zcLJKi+85)7voIC;MHFLW(L#*~!blw`Q9{vDo9t$lQ+zK1{U}a^Y$#4%xJkb45#3r6 zFCAt4roZeA~o^HK7TN6 zR*@fUKB<<`?o@!8A#h97czsrR_~UXyo(0MlKqnWD5bBY#j+@T?#T9ufb~6C609kap z-KbTXYQEev<&%*~c1S}M9^db$v3dkx6A*bnA>qz#R9^~*5Sa{oP#|o(n!*#3Q!C!! zna;4eW+iqMm4Xy_fI8H$oYs6gJohxV#|)a!x=FMG!X#}L8ir}$uAWG1kU!&bfJ(#q zc2o37_x?0Gw!a0Xu&9fbsdwn2xypSJd-J8H7Xj_N#eub9hfB!B`ZWbY$@XC6IXg8a zK|nUxh%{>MSAH3#+#Zo-{<<*p=Gj`U?t4ZZC}HsXFpaBI&!;&p2?R08cxp1rsuJVO z-=HFwk&hY|({-Ad{c>Fl4aSLBViqe%Bp+D4O)cRtt*Jf~>GCx;=buwj{KApMNp;_! zYYfRI_|vAWH!hqGwH|vMDhPZQ_!Yw(d$L^I#VIdoJjVH{DYZ`G5&d%rq2TeF0gMr^c?IF zHJE;~K_9K`MHTpN2sy)Muemk}!#HBI9JQQ{S~A9}nx35AxDRq!Xjl$3-||@5v3X|G zI(>-Bu#B=im?@mA5qZ8=e(Osh7b_BoPLF2Wy2;BFDsRBTa=OuM{wIBz{6U^++Al$F z&`12>owb4iq5pW8Op*G(V}QXXY0W>tjkDMIjniUJ)g5>BoBb}NGf>3|8mydb2*n%~d8b)^tb-&#&KN?}Bnv}i3Z zYpEYaWjV1b@T8OA$uIZT%g{3INx_HDMUI@f!1b7;!nmTaxIaGaN+1va$#UoF5@rO= zg9;WaiI~!hA$k7q;|IpIELGHn413stzEsY1{Uckk!h4s{UOc@+l7MVFrmpWJ+VV_k zcO5=Qt5b<8QOVJv~=fh3v>~y2X82 zBA9L$uK6c1Y`bxRouXSv==eAO5%&1S`wBLkoeKtL(+b_o=%0hvjV*2)QXSVJ6)L{`IxFaF1{IE&+j%(g~kwigp*SgVZ!@HE>=OXyG#2|4EQQ zN7+MB7jju_Wa^hMxX_A!;-CS+(P=(O)y;XbOy}ld{oB8j%3>TTYmg8QR1o;4(a`}7 zY4y-U^&Elc34#-0Lz0@CW;8zYo|rHpdKj%{i@hdp{q+Cw0(e`GL_bkE5J^yWl!e{R z>v52zJ|8Hfa+Nf^J?O5mkxFhN=Wigh-TY~W>*ap7MO(3QxPQi?U=D?h(?4iQ)INYb z%!iFy&xv8SmiX_I&CgXtPJB%%19>Fbn^cPFme&@enhb-)WnbKe&o~EjmF5fb&FgXV zRmLep{9TujM_vL$(~c-?HM7Nzqko{5TNb7!+SYmkhYro{9SKT_n?3?5&ncC|W-V3r z)l97BF+EGbIL1sGXijvY9ooktW8#k6cTi)5dK{$P=9+ce&W-!G3NdTRL+5EzN<;*< znj&t1sq5)*cXq<}+m8%Rn~R?CWt(rY-5^%!+iXm<34g<1(t5U?9j&O3D@S%kftZd( z)sr2Mory=)?GLTkT=yR^r+&ME#5hB`bwck&AGzao~ zEdZ1{WkOw*m`=N>5HyWl_OGq@gPeS`}$bdY2vW-DY^{iC@^#WFdDiH}YWc_S;ga4fv zrUb@wQ5fBR>cLVBcaVwa*-#5+TJ1!vkcrUQuk>+sB|1^Ptex;T-%Kgr4hP~)22}Wp zM6w0`Y}Rk~$qI1&J&-2&Ez1n~5m&-G;2l6Q!DgRlNZ!R>(-%Wl0P#jF z;uh~&#y&dLsD9j-RY2lM;l!gSc)25-OKs2Kb-hkX0a2;$wR#`ZrbA8h{BNoau`bHx ztEXM_ucPAu!-P^U?Dj%=hT;e*Gx9K2jHDuOCN7{_aS3mTMBp4!DISiXv8$lzuMm

o_1Ez?s9@?TupX~UR!5VMcv|Xz3w!~t{t~rr!GGr z)*-8JA6-nK3$Y1-^DraBHS*oe375+jiLYE&qV|&^aK3QDTgv>yi(;)H$Jzl)=LFCB zIb1j??Grhyr&MJtn7F^aon@~kwV(9ArP0OE8{`mTca%P!0ESeBSE;pwdc6UN7;3%X z*y9Ch#d=@+2N{n0bW%$F&Bu}DfTi}2OXM?e(B!gWqfb^W>WX{=pAM*xUU2xu(ciu1 zyK@aZaWie{)1~CVCwtYNk;0))KkY1Sg>X!zVlRFcjnXV*aF7nhr7jRNkbgGmz}Aiv zM*^X1CWOus*Y|(LFeRszf7AmK%q*{j)@nj`ogkxyz^~_t_5R=3sK2Z+9UleE6=SL% zk(+R-$$awgimjJDkoeHZ!UAIwBys#3SEH%fHfp>a1nBL=q#`lQxz7-I6)p$OF@t`= zbC%u_`U{ZkI_)eV6u_h2qwn^G{T&#Z-fkGYjUDV5Rf7L4rD^StO_2w)kdTg@>04z5 zDTqV0dSLb9xg&hI-|neRv+%FasZYk~qo|=|@NHM>dVV(ht})vz9`tvke$Z#l34S5w zqux@$ZQdj<>tT34h?6(Ed&NRL8=Dapv;f(ZuMV<~!H^37OmoPyN%IbV!H0?;!l+sI z-Mif+tVf_@Sv%H7{vjL_bKCG_)JLva6*L zhEivaC9+!?RlDinCl!uE>Okk}@q0ht50s%1$J*k=p0~sWSum358rtP}iP$qe&2Lxq zZV1g^9Zc55HThl-k5&XdwWbpn3chW;rIHj3#X`-x!rCEqU zZVL;@VAbUi4{sn5(cw=LS>%0@2uSjz+jW)7^DDSH(As@hf?lZoNQ9~(tw<+)fRZ9~ zSnB)oSC20{QcJwwehkWp@wb6J8WtZMpBJn+t~-Fp98y5!%|mZjfynU-xDN>}E^Y`i zqPO%51XNu)jR!kU$9FQh9`IE4O*xL{5gkMq^^?iV&B_xetqH`yM>e-U_u1@EQJHD- zof1m3Pju=6_Uc_P;xno?9B7fw3oj7qI-6gz(*L@-{^gem#;m7e4VTdh%2vv-y%Wzl zb%F*jYG;SL#7$4%^PbFAhF%^FRa*3OX#C4mBBJjGPOM4pubqW%%1Ji2$B=ePkwCY< z+0XPD_3Mv67dq+OQtj$i~{0+DN*eBIBKF!?hXB@R%53k=flx$h3 zDW;vws5+uTjc_P9>U~^1G9F{qQ7Un1?i8Zzn zc{BnYCQ3y|&~RWa*|lB)rOjg~CvpPUNWuol6~nYof$n&1p%Z={N`5$ZGuK_mF%wvu z54$)N@EiZbXWE;635~}IJoi9vy(3QRiWq2)XY|cPl^y((h)Tr3=So4nqG@itFqI~` zr4r-R6x1Q;y8ug0llHlEOy^P71WmU%3 zZt)~XlePl0%9UZfp`7dMUwFW#s1Unk^XX`3S`9y}JSw8xe|$WlU1rg@ zV7pP;rp3}AVC9W(t4LQbd(H926F}}Bo|Ku%rG45bBzQ&EkT@4q!qceu zJ`nw$srtrqOR?K}vq8Lx)kw%GkO6TS%q*=TwkpX!4_fdlkN&Lx{@K&~Rw*#*aXew4 zE8xPc%9Qtpj7S?UBSYOR;k3%f8E{su5QQ+7HX(K8SCAUBJbc^{Agbm7ap3fOp{6>KU9 zsFZ}fc<+KO%C4H2oAL>adwO{_Z#MX47DgU|Q0t|tJFV&qTn_qGm36l8L(up!oAw5* z6-~Cr>Hfv;GzayBA``xf)}5`0@A(fd@~-XMz!mM)VWiQlI2Cn`Z}6K3(Z_0-8w1|4 z+T$!$H#al*Kl29!80HU*6w-HOjmy09Ot%qA8sO^G2gjbdA5f@%?*+TGn~x1+i#lpT zM}H#JA{`?Vdsm!34Q`@>}BaKd1O9hT4t?e!zbVJ-=yYWo1obEG05T6Jb$u5R@f{GA8uN= zjm8y{>9_$mZL>CA7|5b^(QDi#>%jpRNlfyKHCoQ|Q%GmlC0z&)#W2T9pL1J^qibZ= zZ(1YpTt?2Swk^qyQ=`JLxyO0Z5effqkQ~nW;8ezmoOgU1R^8SKZljAET>o|FU2lfJJ`+U+`^OLy2b^{D$XN@tAf7WXSw?sONEEE(R`}Hu3 z!AqqH4-AIXKRAB!rw~ph6KzU7TFH;O$%6UJZNMsuC$H785WU*%9!igPks{lZ?)E2B zFsyS}S_kN+by?0J^O^xGPvJrBrdLwKr;oLuhY@J*>cjqbc1T*&-6IUDIF69s_Nt2= zk`3x(wim;HwyWBr?KNsb8%@{|?;>?@t26LHrJH@spuE(r|y-&-6J zK~iFrC)`qO!ld+Uuh1-~CP-%HM(n}?n_2l=yN+?iT{m&jko9^+-$p<=!~XgB?c*6{ zGLJo&cMJ`sJ z7#cSu!CspEG8`iaX1O)jFsP(nYzc~o#&=khnkg&~f#l|IV(sg``x z@VArM;6l|2RohX3=T6x?GM^c_4Z)n)WB3IE=Jrw-Bv(OvnAV2Ut$nD-!W8 zJBI%uFx$Ei&7iI^Z%$J?mD-@^4a@FDAEx3|GM*7pg*S*_z^=1XG41EXazcj#54qH#EbDSj zyhB1TP#9mAsUh$1l%=yPEr$ZTS($&2A;4My^02WzGUQHEGy!j--F{WXcmIpKw+xHw zefxeDkdzV>0qItd?k)i->F$;i>6TDBlx_s1ySux)OPV2uni+|KXN|vq+{dw>`~BYg zEzHbX>snWy=jZ!9{~#ZBH(>{1(6}7?-=7yW_~He92vv?nrfHqY>67-h4#xSRw1#d@ zx-Y_aeU3%;A?dVDc;SAvA!{JbgvMzdNlxbCy|E{K7}p}E`R3$ruMbM;=-@tHj&3pZ z^hil8XW>(PY+KQ#w}a6${ed1YYp+h$48-m1f%bW7*9{G z$SFEjw71roIW3vZvlzOVWrXsAW8ScK)y-d#d{TXR{H*Z>{gsVxAFWRjUnM6JMu3kR z;Kx+f=afZt4D>k%QsXUL0btm7n^t{ITtMFq2U*{2zrzL%^YCJQcHNrIu`!#{?f};n zxusA$+%!Y0?TO>G#F2C?O{W~N?p#}Zu0MO}8!LY#&@x14{>^#F>nXx3=YDLpRo8IK zzOeW)qCST$9Fb9mtkDkJw>ey#gC*FJG%QtP=R21jhwr~y>y6pfHw{EyV^_POP1HaY zs9~H`l|mD`dETeKVl4M(k}2HqDEZ~5V|a@eBO+SI-@{yvR>hYrf8Cfyik8Z??6==0 z*r8UK_qc&EgorfD)#YTD>@iQIk~f>QU$G3?JxDlykkA$o=4!SkpfWV+E-lRrBSNwY zmisdD>wCv;J}HE9>GF87%z4vzUWmv|ff~D3+*-GfDvF+%o<+c!%WuKRU6E3b_xIMT zsq|1alCqR4L4Q=8WAPC;cJVz&6_iAm^F09>RKhG)qtgVM;g1Qx%r)oGB5mtI{F2HK zX@nd7&Si>?87SU!!V+(bUNb!K$~C^&prm=(zXM*)s_Vav_eBANc_`qyp13f4v__ZI z&O+R_w4!xO)%9GO=-o#>AT{9DEWY0~XLYI23`~*)+R*S`?+ZXFS6X*P|6teDC%nQn zxmpvWP;P8--u$$9j56)EmW?qr`YMy>vC0tFsT9p*IsB_3Gbr}LsL!iH)l z+F}CYrOGFjgD(!wTU0`HRJ%MBrJZ81Ey>QuhKwzB>% z_r7ZECH%XNq@quxdu3vGFW>uv5%PpEer^Wmnyq_XEbG zOk?mnfoab`10bW_Mr-ILU7v?XpoHpElfPCpcHWw1lWweA>jfdvFztB8blct^7B1eU zOC^w;Q$1g#Ui9O8a2Evw^2_tWBIt{_uAnDcxzd**BRlTTY$~CsHHNjy-r%;RehfvG z9})!v0ure0XbJpm^@)@XLb2|(QLzg_!|~^}R87KG8&**Zx=ar~G!FuA1XrU4tYRPi zw%pWjb}9P6AZc`kh?@a+SnUebyDbpo9kz zh>xU8;l`m}yQcy9#JKsyKb?otAhL42yEm&@HbqveGXk3FmK9b6hi$MbyQZ-}nW$-f zG(vT;m35%Z>v+F^QL@2xXf6OX`*BLth5Rfl2|AV6@#a+qD$jMr0lJ-?0$D&5_M97f zs17In+fv~Zre!$>oW~(wBq>Gl(P=8LQ}QELs5bFFq4S|&iXb6U?byIKqKbgiz|Upi zgV8U4Eijq}x~*|**@qAsJOlnDX7a8Rf3f=xveqJq@(Kwj;0pf}3Xkt`ccu#p&(oAw z3qUI9@eoeyWV3-x-FpSUrRM$E@aOKvO8V1(zb%IN_F2N3pupd6?;yTS!Ir?dR=la* zD1bzYm4AX43W=QiIG?~>B|Td^L{Kjo{uTqc=j=+FO>{dG*2kxCG9<)rd7%RSld8TwF28v>;$ZXzl{Mr7owFq|9&(Sw!20k)-;D!YLnQt&aH~Ne;A(~XV*hF2e?PM%aE<@(HVo=qGmyUkl0`YtRa<4( z4=EfbUDM6?0LfaSQKETdC7Dc6lKHbW*{R8+Bfpoc({gPOn4?Vi=w}z(u|uO{hU=r|3oXNR|yhuaJ%m7UnTInaeLP9mM*QKOb>8$-Owu? zj^`I6Q?1%kf~-Fg<}5dSe#NR?dC{@r(83V=yA)+>(KQF?0MLd}pUIR2i2Ly@rfTQ! z-2TF^jx4G&fRc=4vOz0HXQS1#rg$bhEV@RzrXO_YJR!&B2h^aX#6ZazKFo1n*O2@c zGd+aJs3Y*nAF`j5!nU=jpjmI>3LVP3!wIwZwHos`a=oQQAi~GsR5;x1*edb&q_fj2bGOF!zq8BJRz)adjxdsC!ZG#=m31>mh02U#m zuC^jz;KsfMCXW0F{XfpY$IZ4|fHfQ!4_rV~Nltj07fTHcNjhHpYL&QS`6%wK|l$}YKff_ zE=G#d-v~q;diGRlcX*FW*Z`15U|R8b90*;6_GlbVFM(tAK&49vXkE9>GU~NPH2~2U zscw3I>$}|5GAeTHfE&WE7qW^anm((NzxJX}1uWb$#+9pbq@SeO+u}NPU(D=f%{p9E z*ewOM6%O@W?7pcDQQ2BH1aDeUZtegLp^Avdwd*v!KdIU$2TcNF=J79iwBXTpLJg&` zqgx8U$;V7)Dwbbjo8~tBO6u?HG&D*-7;2Pii=VW6*TcF26{ev*+`HuU>&C>&Prq6n z{+!E@iCRns!>ph{^%qT2Ui;~!5pthTg}ghh?Jc4it@lc)ov9bFS+CFLAXaZ^3PsyTU`Npw$7P8K67S0tT^a`&ZHSY_BM*rRPR|zckAX%MfSc;t%aU z)@u>31;rs@ZeMs3tL4+AT7L6pIzTdlJ-e%aOnI`#O_=U#% z#=%@Lj-j2Q?VAf5ei}ewOsi;~KyWPGLpaf!Q{U|kzgtBXxLwN4VZm5&Iy1u~|4i&3 zg#x7N^+4>q^0EN{;t_=$3x)3;WAqp9QhhF1dfb=>ZQC$!%6C{DjGk#!9K*~GCl zrv?JW=W(a%j;&1}rlv^v;k9ZS+utDOl%c-ZXh3R9V<rimf>3V`Ve$9C!2=o__md_tG z0^sQT0qPN^pda%mf`?YkLk+@*z5e?iBAzqBD61eAYs?#YCV&*Z(9j!_ zP;3eLXxOM_PAj-akQh-V$@%04b~atRaWPji4mI!5UKAIaHa^HOR+Lp?W@+5 zPHQGaify)0L8CJdw&Wu->@zow66i2T-KFvge~JizGE9Nu=(6&!$n3DxD;&zi?)F7*TT=zh5~DBI zn>f{kUaeTX5xtF!C%V}mNN+h+i4nc7rC+b*S9pnrVfO8vq8`W%Yu}E~riO0KYxkkt z^F%%elrTgAaTzy&z4yIY(w~-TRmcMP`eLUbFQ$zil&LmABGBxwkaY?34PRIS7w)Wy0{Q#Xjt+!qUfy-MOMCWz=T8xZ552^Z!#J5V7T3RriDrM6rh{Hi-!MYgS_wI-Ts zXCt0cJ2RCC3d4O!B!Xt=$`7~+u(lE*&-tI+vwqzh#dAYl?Lmc1oVRiKqCE&MbqqN- z82Vm46Sw95II74~#~sy|np zJ4ls2X}Q5ZL6u&%vwXph6Oc$g%xuJY)kuTZmg|mM`N~`GR5cph#^QaX6UbUbTtSnX z*gu*TQ!J*<>a6c8nZ+q(X+p47x9Qi5){X80G8?}v`mwsM*M5($iayaTQBx9OqJyvc znd?&Y42x(AShp5p#nw!1)1%ZgHSIcj8xw|fN@#XsR!dwnvHH#mTtoY*17Qc0f7GJw z!sLxFOMp4tGz-u^@*>hnH>zogV}xsD*39b{w0$43JdRH)y%V$O5@KS?p2R| zXGE>~9gT(adBI2vZ{16sp)X46=Smhl?2%wh4{{D;=F{#2ClQE^$@U52Jd8yVAgy-> zs9i{Y%Ci>bl)tclV;&?{Gnn!>0O^p%z$@Q)Kca49Xs=jg00xPn<(#J@jgL|2WBL)7{3xlJ$h}w2 zn$!2d`6FByYoT1Pf?F7GiVaCVz1QiK->T?i!&)(Pr6O1f^%=L*1!3L68te*8k9}0~ zn#dq~Vk}Exy{i}BEm;4IB0yHX>CDFwng+|_@J+PtBRaTE&XXXFluDQ>?0Oc5BYb$Z9;5X2&g#o;LR2ZW7V)QG;A!7r15;Y2v7Xx6{7DA(a0gL`1oT%My-MgJd3ehh&3*_V7twxn73|nBb#39 z4^<5)tTsbEr>OwgHKOQp`A6|05t za7^i`gjktcc5a#&`WBQ+M5cVQ zF&2T+w10Ubha8cJ-yLXtv8Z9GY%kH{iaKN94Lu)~gQFxqeh=>-t9=2)&`7ukzZ<=@ zHbIJ1Z##KmuVmFNg&zb|^O8V~fgNSfm8_MQcR)X0CcEe59xwh1;W7BusMz6bI_yQm z*|`${Bvs=0%Fi?Fl#c^YNQN&b1P9v7CN=;Jmqf@ek6dW}#Y{sAdqR20H8MjVm3*+fF59mr(B961^$fh~~RCnz{AO9FD z-wr;3vP-{-gbN&Ef8r@ub6Z>O-`3<{=qvPyELZI?RSac?z;o0dH8(`zQw z%k;S9n}{GiJ3(EYt0`(~mcPfGW4GN=)CAi2IrC9IdJKf+Cql&NY9;q8)ephH-sP;1vhh1$n5f@w~o^sRCSz@}f+8?VDol*C+CHY}Z)N=#x$hHIzdx7f97AuFGe2 zP>;n`OAIB2#hrF{lfquKe1X|HtK0Dbi!rPj@CcZZ@f~_76xL- zf8q!fMvBTVcjSW1R;wLvs5yQ+s(l_=1iDOY6(dr_cIHpy8buqp=l%P1ntE`&ee@A8 zO|+!3yL#5+AyWepUAg)C>i)SG-V*U58SP|pIsciHmcV#!w-w43*hZYk1na=ee=UT%a>TeA?|8@^t)HfsV zMV1ZN-(?RgVE#aL3rzl8?U3O5bNiiDTt{|80eiVhL4aY!4nt-C6!OH>;LKtOM)d20 zQv6WbBlGWUreX}*xk~l@k8}Du*%h=YBB0O2Ycvby>mTP38*de2geZy!VmLyUA$3C; z51#%W3K8V~7fixHh40a>t?(hVy@66A>}RhaVw!9+m^5zh)P{qRYqmTS(i|(wDY}YY z?9HNYG`hjxd3}irqvq^336w0OAJaQpx$nvsK1hBWN8QgnLa{Kq>b}L{x0C4>4ytoLvfpmU2Btsl%iqKpz_Vx0BE#SqH7SJSlF{sr)2hiew7u?71U76FJW>lh7v(L zKmkgvF}=D=2jT&d^(RNcWj3*^TyN)TZn%_0o(A+u;j8sgF1<63zyzcx{O0M9Vr%>< z4@bfQanT4hhF5~@u@Yv1izxH`BxM}S1B@7#FF&G^S+|@AlG69n#<6$=qT308@e&gs z1eR-`a~K=a|M7WiK_Y?kmV}}!lzUwE8Ol6)eeiI%-$&$Wb~{p-HXeAOy}K!%BK663%u-F^*T74N{xTA zi#9#hiCl%jeEY{_JUF<_e~xz-cVeM*MJz4s8Z9jaM?jWl&7JK!LO^vMDCd>514l}wA-2W?it&)-iuUq0)9P}9KtlR{Dij#IPQME=22G{MWe*BAhI4d|YRsIe$IYD>3wguCB=qe+lyo#Ir$tjHjX=gZY}*+jgsuSccQ2Tfz0njSOP{V9Yrq zE|U**TdQ(TJlm!GAbObhNnKKMpkZP3GiL`OGqZ6+^9|~oMZI~XZ5IX%qzFq4P7hRt zCVmj5zqK(&S~+T>uSCJfdRJ&0u3bL3oMv=0#-L--Yb!C$A@sQ(F8hAJ8q2yKLo`{! zc8Wq9-ioc{`MA756NddO$cSvZm!kH}`U_fYJNm!9`3P;`e=0Nd~$Hgk>9Ll*cuLj{NX&27W?|ORG|`<)bzIKBP06gwwFC%J?UFMP*4>QFR^R zQFlbx0NnQ%E92`pbv~}%>qMH$>|c1RvW&SZ*BEzuz(j64UAN@?->x) z)G>EIYrwMrdrgKV2Y5LRHm}B0Rzu}C2r2@Z7EUNS;PoslhovWaVHpEM1IgzEu7(I2 zIJLY>f4($7n0=w^=*Uqsf0a|z{KZ<2TzW_H$PM48v&!RILARl2&k{aQ+@Fu$S+()W z&b&Chg0!ui7FrmMJH0jZbZr#CPR&~#xgt}gzJzPvb%hwVdxo$*_GTlr4Tz@%78J2J zTICm8evap8OcQk#3no(h%~ zI#1Pu?lekt7$Vw#+>56R9@2%PzrhHB(3wEOs|L#EK7v|tDfsDS*2kXc zY?Snb+O#z7kZt=o2jQLv%MlSPpRa{Py&`+jYrraoZaz#=o#n1SwpXHrOqzBUVvD=| zP9-*-Pxzr*{^P~$;mw>~RN;fciSW!FiMF>lI#jXkp!*UYbF{G>@sqF6WGrlB#4#B) z5pAg?>MDhdOxP#XG(#;m`^h`f$j#??;4kmCKFUVc4|Md8^~N2t1A}&|)9nSgX~e?7 zYSu8uQkVd|wa}SL?0EW>|8w`!`w0=adB?OUdZMJB>DH#;FF*=A<35>!dEKMQV|Nq) zb?qmt4_@Syk)8jL8ydN%xORwZR^GBtVjMRSE$CzDh z+oDPiInbwEI9j1S^PQGAr@cu<$&Og}aRV{<75maOxwT|n!8W^g*vu~|O~}9WJTtW~ zP{=tr>HcK>E@QByw`ieP2amgQFYVV#esZ-xkdR#VBwU{h>Rq>_Kh{GN9;gH6(3wul z{D3F>iS~jA_oQg%zj}*9AZf`n$^`pS_uy2iw(v`Xp(F*51CC8t0p2|Q%E^0my=A~U zqp#V$AmE7w1CT=}S>$8fWr+mVne*5BC@<6|-KvFgcAlxt1i-xtjJn$c1o-C$$n-yen~RA5wVC>QiCI&Nr$6rd#V-(nKNs6xdV?*(nV{ zA9)P*ZG|=ZDJl0J{~X#R=j5tSH2Gx5qs9=`h^(-3%w#8zz3x7Erm{tAlX6U?BW~jM zQfq662H-^JZN7T4C8k(K0n%YS(SSbB*wH^n7|4LD=#EW6Mou(7r(U{{@7eExhBO;4hE!g-n zmg@&0KFVBX)q-oo`%FGgpmqm} zRvl=1`_0?Bi6y{SS=*mhPitdn8H#ugBMu*nNrhYinCwjE2ZT{IB)-%Me}=v8;r{jz2xg-sqHR29Pzt(V3aSIl zpNftu+TLY&Y+-!tct%&ty9(r(rw{RT%3WP3TFfSKER?C+M7U-g09LUq=0kyqt-v(&4p)!R-ET02Z={j$-Ow{Qy3`+% zBC}eiU2`mwL4Mn}Nq*22JEQZ$dsGy2MS+JPb0Dt-!k|0F$K45`(KYPQWIXUJ|Lxjr zm31$0(gs`C13wvQ(TKeh&mIEnPpxyRe4F=+*AA?7SVk<@i-EGnoMkch)f0J(OI8^_ zqlLCUwLR>5gZ;C~?>BO9yr(kRRJPHNd01Ql7F{mOWcX7$P4xYuNR*c}shp9x5jjsv zf=$~nE7C|yDqJh&tlU$E8$v-`L6`ukxbk)!Ue>OkWYVn-RhV^HG9g(cuGu?~GduLF z`%E?1;*k#C>$U?TqCg7T_HSDF4?6Q?GTazj;b|WpL9^1qZv-^IO-$ zY6}<=inDOI^NGva+y_HFkZ_m7N=Q9z|RP7`$bGf0TrlI$Vg@4<=+f!_j# z1g;k^q>P(z)?uPom0s|uC*=L{VGo&J;l@5`Jo}c;w|A74a&5?9d|K}p4| z5!$Cxg))v)pRi0mp}6hSR>uDp$zggp$iiS`F6)mTU6)Mp7XKmFHVlxn-!y!ld7NOM zRHQuJ5&neYS*g$umSs?~?-Pi=r~WA4Mmf4xM;3yJnwue}1ce|pxhjgJFHdZMm&A%1 zfIGy@N8f%NV3}vyhnc#Q&bS;3Y$c?gX=j?00l^jFpD+CXs9-L=q4M_Sz-zl-E^%cr z+2S2o1-QePC_inUhtO459M(S;LSFG7@me^lXrUkIobIJ?|6+&A3=n_i`U#Dprv6= zB3lfV`tqYb`Dc=e;BF#%^;=0>cTBhroIey4UF$z{B&(W{3dk z+aQMm{cXW@XE5^gsn3DQ4=p75V2}5Hfc$M1FoL;4t=A7}%yb`L0`-%YV{@5Bt5t** zP(IVX$dK}V`uetZ&7-q`tot*AoukbK9-Y(jE4{em`l%u4bNsJ1i)~tlYkRiTr?d{h ziE-SWF3XdfiU-`UntGVCd#79K$heJ8V8d(ZA64JpZSEz5uTB57WUjPig{of>2=h8r zpF|6?|2@(>{d1(Jtwbd!IR{%DrLt7Fe_t8^Fr5@VPl23sgjD`ULzcnTg3X1E``y32 z0A@^LGlJRvtX2M&vBzwIt$P3V%;(_~Cf`j8qZS zA4LjjkRnKnXB+Aw@0ML#=AKJPGD=mS<=+~raHpiI4NH!!qQkUs@y3W_e+mN9s;t7F z8Wejz(^S&O#4Emv{7~^J7I|nmBJcK8;WT?l`MoB9mH5KoxVlHJA*E&OM)kjnwX0hF z3S6eM?Kr|h`Hh>qgV%M71QiTbc-|+xCN6%g#2XcOqn{6%iiel^+s|2U%O(B_47V{E zf{p7|3=)mxEeeE)h!j=5r{8!kj|WAO`RgVVi($eOR^&6Z|9;=*ZCx?lLQ5&Q`3q2SlErc-<^2nhvqM8MUwPBU6@Zbs zGy`V-K(N2^I5UF>1A!-SA7yUbWO`X< z(l=Qj#SqRc1o(QNeC9hIB>IfvCX)l6cKZwb36cjCWTsRIZ<(1+G6=IR5gA#thP5 zL_F@DCq|xSkW@?US7&e~z12e`ks&lct{oBcJpw^3)bBKS@i={JGN4+N<|rs~v$rVS zBU4DSp1m2Mz*n2akQD-DWQ++=W~6uj0Z}QD`d2s&GtZ&NdB&6gY;>_P1hhc%KkVVB zB$-Fbp?sveHG6s*5~HDSG#Rt3ti)S{WHCA@aV9&-3KTqDN8>+f>bv}ls;orD%igd! zY6ny}r(b3bejFkIrslz~qWkD}d_3)f(VaJsoJ9Yhd}RPS@F-D_7{>H~c0wKhUr9~o zlJ}-9gUlg$Ux5kp#mlLJ=+1ZB+o%5*8j}a;F3Z-m&Hx@}_v(MAEwTUq%}M?*Ozi)A z0n!BHP9wi->9*T@Q`c^%fg(8c&3EI`x5XLw0iq9gYQ8{GUIAM)a*rKk7hkm5IO9D?2 z%?czNZGT0mG(;b6N^T}Pl<}y4Tx+v!h-S+h7f2d6thp&uD(hhMRvk(GMz=e5O!3T3 z_=|$-ciN#WyPxJGH0c0nros&v)6@aL5TCW12?_#Lt2>$0=`e|L$e0Ubc-4W`aRT+O zQxt{gk#O4=-QOIsx?adVr~q8khF{n|`!5~9n-k&pAj<6_X5{+EBxH~MTOgTFm!4w@ zer`V~DOt!Q^0h&6JWOK7a#RyR{3P(7iE*18%YvH)hh7CRu5w1TDc+8M+%47s{10Q4XOv)Z@lkox>y2}FjN`u;_#H0-JkAG(>bKWF zzgTbo8`~5<;s>zJubSg>r|5UU^6~L#zC+6U(!W^tyRWmqF#`tBBKJO_6H-BwWmXhq z*8r1OBDXD5KpWh@3s80Q3*m%JKXM@Q7|Fk5_;#x|zI1_tVnpVdxNWCD7x3`v0rjV@>;2+AMG_0$R~J`z0K@px z?*YyWS2*7SoLNh;S}*^Z=eqwr&lRH}w*!7FpdqWhqMC6&gEbF$9Y%kvt;}4TmQmeR zjRoi+H(DLo$TG9B9sv*FTfF29*39y0R$u!(gAD?x0VTNuNvcTSuYALiY_A9cnmKm>W(72JX`ST;4N8EuG4IDq z<+M83$Z!JAxx(#PhJ6HV>53%ZXsMRza?LILG4l^` zJz5x9PThdGb48o}v6Sx}y&9n4Ek-!xx@9Kxx?gcnGVBl#)B$}={G?`Ly%^Y{3m65< zsUo+0ocq{iAGKV)Yo+m?t>Ujmbmh0fxBjcFy0wftLsZ63EM?D4`?z`T#rz-O4Z8Pz zm~b)#hYFIkasBf<+Tji-(ZWA;m9wz@ZE_x=A#Qqv$K#lf>9LBHUk9Ot*&F`|U_#1h^TErE9N z`=SQT>Y&-v8&8n0w)Zaz^y?noEpYpT;RZ{SznnpPP++woZp9?J>jY0+G_@)qy-W_m zhS@}-6%8QsL zKE^xEVXPA+x2kC* zBFA(7k5)9U(*eMx6jM12+S(4t6nmfQ_k}-Kn{()pHc;KAexl=b62&|Blj9MwKxGz2 zj-l@fHn&?hE!-P8`L1U#VVzB27;+OQQ^D4_mulVYWf9rwJsU8FD9eYyS zcJZ9;?Be;7`B)1hLu0e3B)T@>&L)WwMz${Ezf;99i?Hy$6!$aqJaOf=Iu#DDWBU(9 z`V%azEszOtCDWv(sr$so>0ZNf#hOuqwK6KsYa$#UybwpAZDo-%wD(Y9bXh?5&o#ug zK*d?M?8YUE90PSf5VbDU1w!T^sU?y}6}5j!+85%V0S{B}BAOT>>hEA{j!VFUtFQ^S zTJX|(QGy(VnW*=*(gO6=qbYWqcx%Wlh3{?k*D{%L;ag@6zdbYg;nnpIzU>fD`uV?%{I}*@afmtuTfi+#I`1mTu+EpAZ(L! z+uct`#U!rI8RTot5-cRj@4AD<5ys5=1M+_m9>_9`8w%*y6x{E`XYB$)x$Rd!CXbk_Z32r!fRqKKbE$XeyZ99g zuR%q5`vvB0wFqpx+Y&n-o6|&+a@Z-T*>zUODzp4H&}vV#;MD)jWa660gfR#~N*C+2 zy%t5bxZE4Nxow9)1^F*LCdDlfhoD)eEha&QX*ks6JmpBGF^!G2^{P76d7+I7JSr?J z*vRGZfwi)A8KqtY-V6eK+EinNpp9-Hthe&VsG`$2w>iZ%*Uc&jP^3v#JC*7oeveWfpdWemap(~8-87P9<3Q#DBRYS{z>JAv{@w;utPBz5)YwYcO5{ZMXXyWA-jozMPr9qtoC zo~-p4c1h7Evz=5ut_)F{dqyn}@)L<1Zz9-iMeII2yjuP+$UsttKxlh#96NuRCz7-R zHj<3T%SA!-2Zmz&0h3S3*mc`F{vi8aXLDGX=Xd1bbztk(d_=|OtLGpiUtVa3*N~ly z4TlB=Xbiz&upDxZ$_kkSdm#On7=}Wt7g3)Eq8s=5L0jpONJg2rglB&7Z4XDtgM!O< zEZ7GhRnCeu10gSPJ)Dxar3(uZ$4f2HDADpx&hlF>D)wE`WZF0$d*WCGjh8IT#N0dW$qFz&&Ko_LiV8HG{;uMHiixFzHn9u z93ctcy44#4{E)F?P=D!Fh^*g=f12Vej~7p5B8aborodj z1-T5iOeti#pDuyw^&Bpy{k_(GxSgQ}>)ssST`OawAxFH{iX@Mh_N%=YqKhT2FK(KE zL9>cF5gW=hW9bFPLe1;r%^#(SeH;p-BQnbKPZNbwHrbcF_&C4Xa%GO`tq=$1&KZ*~ z3H^ZEA5D@e(XS|EGC(G0Jk_hl!$^bvGTHG9Pl#irn6d3#VWR3iyb-YFmQh~(xg^tK zPY+JcJD90Sb1+NU*>?A=de`;7hrY$FC655Uk4vuM4I!mWWHvf@BV_c`hhO;6?9;xk zM#>lT&&x?e4Gow0E}5hO7Pd6qFMqP;bzCbk%O&4Q{585R@+FmCtLvk$^o~%gF4z|? zakPypdCD@{-k~paGOMOO;8v#A9uxLv1{Sk2FEuB=tWCefcTrZ^|C$Cqvyh|6OCVnB zt6+Ye(U$^47q}TQ7I5VZ@5~o8TvS1+mxmd0{BG2-zxIr9qkLi+&(GWw>*e|p#4>e- z3$rSaL98jF_}pFii`tEryd+hD-%ws#qqyxIH(_2r4-VMm<~U=guS*sYWhiW(C9d5P7PFVv{kx34|^qG%GpG8u#p4S&)uT>O9p_i>ST6r6zoZ}?glEzc1u`#yg z%@S3LV$$p?mc~d!d&$A{WNO9ZfD>*2IbYz?`2bmg9May#Unl2?e2UgFW3~#E#QkO$ za2dl(Ml6;zo)>D1Ge*7q|LukoNcS{@RCPPt+Rmy&IS=L-z#$|d*s@EX(?W%9X}Wn4O4>XktX zNNndOJ3+Xj)5{8>gTw_{`O6J977m^Re7hVU&mxyBL*A~rdVaO$b)elnSTu9x4}8Uf zUvng=+pU;`=tF6>T@o;>9~VeZ~i4%zDcdb=NM z?#P`It(bD4>OnNO&aAyJ{l0SB_bSb5Y3<2m(gh&TkfS4Nzyw(g0R#?Yujhx)PPG0d<~Ir7{MggoghU)}yX0 zRWV1!yN*d7MK~bf3fMJHy0KAO_%c)rV=!~Jv8>cC9M;>;sAhQH_(PiD`@ggd=YTr2 zuqCb0ceYCn)D!kdBb3B-=eqB9N zWH_0D?QJgVgd0C*Pk6v5AZHHIwPiEaqojdS65IE0-unt6 zS$2O4E)~;FW=!UHt@eF$;Un`*yY|r97Dw>v1N-(GbWQPFN@x*-Wtg?R4K8bk%Sv8y z6jmGoPz!b{4rxy(8Yt5knOwe6#%f|JTxkR*3>?N@`SDoIM`1atlF$@T%^YOe?d~vG zuXgsfX|yOUr$3Vq!^sKb`f=F6dmfCi^CtKqpTwdj9YHupV1(z=QHlMWK>SMjN)FH; zjBtZ^Es|~=uAF$ z3??fA92UMlSASJCV%+RHbM^S1P9`Oi2!ofK!nsc{_CgmFs>^VH1m}h7g?SzkjJiv?Dy>8SesGmJG3< z`RV+=vP)9t;7B66I{4AA)riYngj2F-vtc0OsQiZfD&U5cP0ePkBf$cw>rm(zoreycMbTc7M!&JVH`7 zDd{uffd)ij(2THdO9r!9SMNTDg$m(ovm6&5t%kKPwS^z(RV+*9{B9BZNANkx9g2H zW4n+KL}po6Ck;-$4LY131A=0u`c1>P^tFzj-z`~IKvvGSHMR?mr9DIDHqi&*Ch%ic zplT@+KeY>y8#+?TMac0!nO4np@l0nwJ-mpdaWO{cG1d=^a}@UP7I1p{!0mUq4eQkV zHW{cfp&%aZ5gkvV_Cf$vz5~SVec)&qF%eVVL(a{L_txI7Ell{ z2x$~45s>bXZX|}02HA>$fQr)H&Cof7(%m%-NJtGZkwc=doTGv{0`bB)OL7*X5Gw&5)Y-0QFC)r74DAJ>&X9&_mOK8k%K8_D&zI)5< zv33^H{fJJZ<1u%!ymNPW%jUKJ?7Yqmg~Pe&sX`xYyY}c;91lvYf_HZ@+rBcQ@Z2^dc3{ z;+JdLxmZ-6D9&hJc7J0qrhkMh#%fTGvc*#B>j>FpMK+b8@1Wf{OH`@xM2u#;+BsTw zQN)l_mKJ?db_uIu)rG+mBln=IG@B-tnT>*>(|s+Bu5Pe0kvzXu%FHP3JSMT)mnFIw4CHG(%=SQzJ~M4gN^-t&N3G+}V%?yC0W+9K^nKZ2Bn; zcR^j=96{zUy8>=b(Sh|k%fx6@65C>Xkfp&s6>wN2b?ZNFx+>@S8f!J+~WL>ne%YzI{kdINdsrf|xI6y^J;f}52< zA;|B$H$7kb5WrA7_}opP%Ra>DttS}Qo;^1YzxE0{3XgetV~&CVl%`e$vJc&D|J=ZR zS?P->yXk!2qp8Ebq@DeYc~a1?)y#|2;NB&|59#t>Miw9WXgR!`;ab(fJ?I8U`hQ)_{bB%@aL#!k59r9u^?2)rR=C(lOE5{T0CGm4Q`2IR7ACqd;-zItKu`iHrP5;1q>O3R9;TtL?% zF}j|#h4bernQL z+%AWC2Q^aYgeD!+LD}$vS?-n&%YKgdgyr9C0~@9r1lrQeQx(10%__;9zJ6_LYT|Yd zd6Wa)HMusa(}}u={!g*TZ4cafb8kK}8Yc|2D`PyUk{iJ1zv6E zo$%AOn?&gr1#7ZocfyHSWWol+1KGr1w5}-_S=BdrYFn^B^Pr8;EC7p{I=g5<*8bu16t-c(qxbdq{WfN%M;68EXZ}EW_-*ddxI-eUQxH!iDK2R)9 zZ8$VF74h)~XPf z2M6&MsoQDEK%uZf@MIO+2I6jS0q6w-Yo~{_bI)&UgHq0*yCx9ZY6O-?h&2 z(+Mp?@+outnj6!|?>_u=u*|;kSJsG%Fjr#hG(&_5#QCTcL@c70Qn( z=}z{)d0RX+vg@4)VDOHre%_U{Ii;KO#8Dko4o@7>r5M+iu_+j74>SxAn!Z~V@QPX$ zrenLHR>b^hES7TC_vSs>kxOrmz#&iIC9M(Ot>;(I$zw;06^InrHbguFYkBA^H(&M? z<8OPppY<6Iv~$1STm^44;k0cn1pDDy{?`5-0BWj z`HID!8VyBc#=uq^y4owCk0Q3o!dV7)q<;O=D$i;hPR$x`UgeAc8Kf;dmc~86X^6HS4Jm&zcw?)`O5RUQ|lZvQ_I|ZQVzik-_@+wj1{E*|QJu z2xAKug4Fj1LO*`Bcj1U%tv;#Ffc3dWy2o?Sz_n7pN(WD$A9xqZ^48ww;N4VVXR`jg z&T4vySY!`MW!Y7d-5Jb^tL}6>9cv8enyWvC*W}*B<)`_?hi5sNHRRY!4UL^XFZ9)w zM_uz_w}0NHkSj68&=yYtCoYSQrt?Lq3az3tJLMnZpTgfJ^Lu8w)SXNiZ^ueUMxvaw z&#mHkB(0qD_#nHuX;l+S8^|}IFS3l2OJ~|GpMP0^+F{2FmGqjNJL|#{g8hM^@nzxm z{>ApXyn{$#$nv`V`b1T#;5S7xO3jpQl3Z+EwZY*h9$Bjuu3hpzDy2ikT~=#K(Y>K* z^y#6hq^qm1y3zCCO6dvFA0F(n2BOsmkyJYZhbK>K)}cF3swRhH3OBFrOw37N!*s?U zZd^b-7Up!Ip8cSlFRX`XhaA*>UtEmgT-~wZkb|FoK+}jCTKcLZEtLhFL*C}o%W_op zWQK~h7%I+|EZ5hR$S`4gKcXD!j^7mHYjuB4THlUw&-SRGHBK~03~{9^fXuN_7?}ie zG>71ixoD)~-8n_~CE2o2*x(lhsU;+CM|QPA9a94j_l)cJAM6&(Fr6%=zKOcT$I8hZag+h5n& zqw!Kn%{lzh5;BGDsUe6SwfUyorb?ANCqr1K+DBE>JDt@3$`0pcuNjf^mO$%C8Ip|D zN7OBcR*qMP$%SgR@?!~KM~U>#u&W(WZXkDgo0-RV&begJ6ji>>HQEMWE#xz@;9EIF zWWDbEcrdpUvRtEPlu1|E?%6`!l58eMWU6azoO(6MD0(k>jj_~{7K6H-7(VjxG!#Z; z7<5N$+Ol#=z=O&d;Wfq8yH31#LwPcxChqzR1o>O_c|l3N2+sqv1wMpZUa-iNVgFu* zkD+V9bDt7yF8Vrhw5XFOC@-A(%jy+7p9$|>!HCSKmr+w(w|d2^Che|u)@FLvn|PeaC&WoU@T*mAW^kQE|c4{HDUc-?XDO?g93 zy5eHM$XLnS<>?BjEoUa1?&!N>4%X}Y8F^eVcOL!fJr`~vh9yN|22d*7J&AgEGN-HR z5^HOkmLg~{{Kg|#boSH>p_9xF9hb_-jjzGnrFqV7pTs4ul15lQAQ^a6HX>5Hg=Mxs%2tziyvXodGQI$4L00mXlSVrer?6xp^X4 zZ*zo68^w(6+>TCQUlH8MJ!Ozxi^(nEjbRt9u^ht0uGaZKl_OkzI$IK*OHh6XOWMC% z$c=QY=NuVzoy_6Cz>CuvWai$r{Eob z$;e+Jc>OPY@jEI-%n+uL&ICc5auj)5XN<;7t3|w3sL+-yt#!qiv;IO%W@5bAYQLmv zYiMY$(FZ!C-AXWsF^Ygx%*pV(rk;18r~v8$bq3XCmx7g{<*>?r|6@f5OW)yl1thc_ zM074mD2;4ovt7yL__EQ{&9_C)g;mR5E~iM>((zR~xxMdEKAuL()N=3yy=EvwcY?Ic z?sTz!R%uDWK<#+l0?0b^V%~)K*-k#v0`5sKu3w) zVyCQM&PS5nbKcIQ(aj!z6-;7|y27au1DfL?<7qSL)I!wq%&?Fov%l8WRvtK+pzG|@ z4DDi9gbL`|&aJf}615*d9*mHOC&={X+t(e6?0@2ESShpl&i{pwY$djnyRHpVGkoxQ zf=P3eh1({2Wn%2GX-;2LgKX)3MN-3(@3Y;N;Yka{-8nqlGt;}M-8t#{A-S*wgZSur z>mp3Q)-0)DzI?bfq&(N3-$Xi$qj)f?NwL^Krq0;IypsDQC-ofh8bW`8+}`p#r)WKS zPQDJDjdCZP7}kI3H6Kl0Ia$0z_BpL!v4bH6R66=bfk*-!)4{HID zOSqk1|J~DM)$2jXF!BfqH&qc|Dn?kPHt^nhO@3?V8C*=nsqIAxVm@ zAGhUw{=TD}(J|2G-ZtdO@OWv(Q$WGy5|?%S<~iaoHya`#qAPrs>?x;7i2JH;OXXPs z84@eDM3c)77v0YUf8`#Dsx^E>F@>jpvCr5x`1(K2;X!L z4+-Cd+%VLSoO!o&n+fX{3U^K0Fkq#Q- zx_uXm*l~OBZNF0uIf3E<@m-W%`is7nA!Cx);S-4=ovN#H6H8LRVFKpuD`sSqh` z5opCzdMz~)(RQEa&tANH``Y0rhoSA_y3;;%jA!Db?vHu6KaLzKr@nb(8s>t&@_a{`^&X<@>m-$FF^pbAS%H2Emyd8w|buXQzG{)^?cRg z1_FD|SU&e!o|VR=gbSa8m?V4^yhYE9d9JzdaAan6gXMP$@!t40-llq;%Av1G3>ZCJ z4G16e(l`D#uSz37+EuvzUE6t{Y5c&$D->({JKx>@JQg2Mo_9v($md+?aZ>zQ>^lIO1TOl%FNloyX>YWUIK8YH{y6tRE zNBPulDnWOqNBgr<&Wf%`NreXDa8*5W<9Ng)VSUl`jmfF<&B)7TKyi^IX#$0 zZh(&XY>>_l-{r#cLMH!1YSJp5QK2N}y`@K_ba$UG4ToCkX|DIhKRu1%O+hi7?xdZ~ ztGkb=T_3bi$=4p$8=fj7KR7%=hvMbA%I+TW%@(<i+q|1?=$qd)Ql8WBW>t%%Q$O*{Kdh$ zDp7|8Ll_k!8M4S|9wEz=f=E}&jvYohj)oOM7e;BKo*}YfYPx-Ep93@3ok_~B{BP0` zHon6)1A7yyy_U zb?HH+?YNW0<`fof;jXvDm$TR6!{GQ=)r2?EaG@s8VKMZTAdtj5oLuVXF(BR{)H=Ru zd2?oT`qf}rwL7sFHy=Z#d;i$xc_%`x;nBr|{#I|BJ(j8VRmn?6aMR{}`7?1GPgI5) z+il4gxX4qfk3W?{J-*>paoDJlI}*cZgQQ~ZXJGg7Hqj%sb-8X`rp_7X`f8YYCG3eb z>NBR;`{r4dYDo|MB~5{DOX}VM?g#Cy*@&ru^v*7yRmJ*h=t_VAd9qZP=EE$TnoF(S z*cb=FVr9-~oor9dLeE{Oi{WLIa-j?=H^Z*k(%h$Qdkwa>v^!sII0MN>_N&;WSD)jz z9ZQlOMe2odqSi=WlSEG_Pmf}VH@K$E^zAWoyC*~Wj`_#Wa!_T}ylm=JBm!f0ibOgo z$oS)}GKK8Ug1SK}sxrGTz~1P)`w#Qq9Ldh@mHzgmUi0Z;!t9Cy;b3uD9CvUSY8P z%Z!KgLyLKibJDDb^mhvP5|8O!xGTI7rCt_^05?N5s|xH2MKIL7Qn6pN-1QDrUq7`@ z@i#e~QrN=Qr4{-evQjfPL`>SsgfTX(%iePK877Bae;Y-bjLW}}Joy(69xaEYi8Z+?6^whw=`;(!+{{Q(G0%BNup%^3ex^%Y$H$O{!Le_slIq-KVT zXS-#Sh(l}p)VEto%D0>K+uA?3#=%q9_65wy1bYKv>}4ZoWh)p>H(74u9l!(L8r^;= z{Ocuf@UF=H{3Y|xU;1Zfz5DeB;90J`ryJVrPR`?j@Y~nH1K%3UP!q$1FVA}8Q%K<8 z{^!paw}da!{^JlUcvrw^Lv+C~d%=Lxk z=#t6K>wHd8f~$R!u3sWsP`~9OOX3WbX@(PVGtLDP7}{qe`T8r=^NOsN5Sr-o zdq~|+&Fi15L80M&hhDEyQOyeO_^xGf5lE&8kJ9U3viQ6T|4bW{{?-QYo?Tar%HCTI zGTbBBo_;`W_{vXkrqV>dcr0E;6B)Zp1Kg;wkUlhIZ9&iE|LsoHM)JisR+xXO7UOf} zpQtO(|JqfP&vZtI73hq{8cvP%uJ!h~MR0eqS!cI5mJgb`y+Hfup`ow- z){jVY)JVR0B7Bb@vNF_OMHs=YYkt*hD_gt9JfF6GAGFTxa#(fQ=vHBv0 z*FH}Btu)2SR&L?sF5xb16daQiUB%=60`LgZsZ6=N)?;@|^z;quz2W^>b*rhoXu1(K%`ozk)CRYYlaO_JjW*$6}U zY*sCs789WbR@3VP#$oyu_VLjheK)sTVo!PByg>wzO-J*(;D<`=hietTW`1b2HY z*GC}Wh7+s`g_(&qyJJ}NHN+{8)*UhG3WE9vdr~}}<*$mChcgpvtTue>GKQ789zc7& z?L(scbf1i)HsB3AS%9t|?+X=sbl{xZw=`n9*PkwGn8GXP5CT_T%}FBpH~@!imZk#5eEs%1>#4` zy$kmcUK)!~u$a1x(i)*mJvEprF$kvE3V-+hX^J@@kzaZ^xPPUN$PpCb8vQ14FWr-B9GcuQ!}F z9z}DcpNH*Hc5JHfwVaGOh|I3QU_+UHq+1v!rM0 z6!24>t{8z4Cj{GlX}1*@Vc*F4GEX;%?E6wci~gvIGV9ohGMi`u2CiqNq?^8t710;P z^o)2#yM5~rJ&xUF?u1i9LzbZv0}1D_%fNUl2t<0*kA2V>zGs4dP*{hlwsLGZSxBha zt{7QxPy7=YIsUZH!vB$!bG8yk4TMh=G|#d$MT_or2fYvO3rPmTS`F5GbDO5FKNO{<2i=!doEol4SEudK1&=jw_G?f?WKOx_{Q$m}C~Uf0 z0C;}k5iS;N;tx)Zr~;%%H0Yx~q6hhLWnnTgAzuFt>JQo1JCl79wll{SkCRLB%4^Zy z984ozh)aw!D}Yx^-b-&z^m$=VC-gP12%`3h7^Y*6);+B08%=4(T?n0#3@&?V{XHvo zZKf5Kg)(~&qyzJGp*4s2K@x61gU=)93n1s1!RB4s}FRcgV5y`p@(ZC!m{tl&RL z`3&xopTS+!M*w$qAb5GQElBak+MTUjdz=(b?)srpvCUV5rE-)d;P{R3p@W|?%I1u8 zQ{&G`eBwLoWDbn};_LW=;DxLRnHYvdNQNMoZD*H-G_G0i;ZqV9?({V6h3A_rzEf`q zC51hp_nH*hoFBBmzEX#%Lz)$EmJ+582$sj#xDK!IMQ`YXFu5-WH#27Z(%atSalwTs z4ViJ6{$Q=JLqha}MV^K3ijyA+r^WKz_o>T7r}Z(XcVH|+d$hbw`p1PVjr&iFj zJb%c> zl`jdZCgjoHN_RULu5TsOA{c&AF>N(fk`Jp}Ez?veahzkThyyAZn}Do|`Ix%y3XNoI zC>P2}w*q;{g@mFxaGgjC zFz&tAbLSoDZ!x-}awbOBEN5brkHdMjtXy$__=)P>Q&HLzexdA@;oR6kCE;N`@8#wN zXT%R^8|>pcCQZkv?Cu*01@Q-{szrUyObuC*^ho(#ZgK0DQYU~F?iM3d#wMbAAa)FJhJ_VBveWr>Fu=*wtX?9fiQD5ELlvb zl9}2KlbbEjU>Iyrg9cV!P@nf)8{AxWwqCsSpDS>JS`VgDsIiq%8}0d^L_*daw>b0G5|ga$Gio z;@wyB`|XU`fVM^N8#6rf1+%njfYxQI7*yCz?1aS9YNRrc=4m@;!Y`23rZuMXmMKjV ze-mDNv+?A&eALvQfeQ5}08{`stEznIm-_&|>H_$$5Z|A1D_KSg^RI6OPp2?a9U~-$ zHD&?t!ol$eBKHONdt(PKdjHh-zZd;?Aq2e@#|oD zp0BfuKLrdzkgRWr^*sEUZ+GXn_}_MfnDO}&UmV@PSUz$Biy)0>d-uw}-pW_`-G2?n zPmuIq%lRKa@_+3@oLt1gIX$Iu#c}C~l|;+q$_J$^jRUDq?b_l3IPYmN|BX zR!SmG7S!8O;M2b2z0Hw=$E+Y#G|Rwwi(~jT+4I$>;qJaeH9iSOkNGPq1Uzc0I*Stb zKXtQ(%V9(XE73hS_0Yu(G<_h%1WicR$FkRVZ!Oiv9)8|B7u81G>f)4wF@zmmY%+7 z_w8N&z9^zIIs3w;LfO!5{4u2kPqsO>XP@1+sTZ=EenT2u{L5D!rq9R?WT=y#jqJXh zpCeJh-oTcah+|_LQ;nh$MVQc%lWdAgk^q}^f`_n5-;CV4; z&vVCh1p)g&jvWzJN7ICjPic}x^Z7O|FH^GQN9MGi0@65B(|rU zt5G<#VG-IOF&Z7N&H*j~RsN=v+ms<@xBkc$NO+$wt;&8g36ZHdn$UlY1*_IX4Kcy9 zRg17BRb_7|w?u}vE<6z?(0cnDXnw~#)QS(BOXS*^yDNx;mlvqwX+{idG03Jg>b3Or z%bSKZYQ6n)d*&l0nn|vh_T)jgZJ_7Z0&rp}42MF1pV3a?dKQo4(RDskRzpLB7e9jO z4?;KTOeVM1BDYC)u04Y)`tp6b&m-f@SN;@S;d1=8g@fu3&%{_yP=~el_D0`V!m>0h z1ad7%P{hJsFByfvygH>(2<UU5k%jLtQYxebHQC1n2Z=l-8|7p6oa^77PcAZ=3{ zV}U61h>UagR^4;vhc7*@s*r*1dF|1Qnl+~IOzw4^Iqr6_;B2ZTFNdYTR`UF_m3|Ki z%24ggkFjMQR1=T>h*1-M$-b{*shITT=&zrp=p<~92J@G21^MfoA zo91B+QS35p{W1(z_2aM?sl0G74K9_bVS$!U_%V*39$+)8aMr%%0>KE8|9vlC!s0;7a-h zbBh=fUi56@f_r}>%+^YNeJu9ifhrBJki6^a*i1nfziP3moE36Q+C$53m_ywZ49vRk z(8zr7a>Q1<%o9Hsa_oHZi#cg1L(j7}TR(nGm}>@Z z7q(*=dOrOSKV@bg5;x#JnLco8Ya|NxK6x_|5|#mu43tD;-4D_g(xrE;2eKuF1vYZ~ zv)B{MI{`;vaORhE#VVqAc3cGKN>$Sv19yMiLRF7^Vtv{Mo#y#iwN2eN=|*abD7PEW zNh0E>Jx@v@k>m@OiU6PCHS#jh(Z+)>-(LCK5|k9*FX1Ac!$T_wetqS6gA$K;RTxjz z;2LKu9hg1m$NL`n0$Js{|G{sa=*C0KsAo~U&0gq4D`ZX;u*d>(^1&z29Z$}-a#WwT zt1bka`J^1)Ly`jbnBt>=^GOqjj~0XIf}r3*8lkK2`&*Xq12DMr zM~;nd8`|e+rr>ZPxJr3%k*d#5QE1n8(XLCA45<`Cv*4}$ohQ$brqpg)yY6TO^-djs zjF^0i(O2TkY>-ma`$EsmmG4shjq48fS8g`6q)KVyQ;dxg1YPCgFziX3ds~e5(PHkl zw`4WsIeO+2uULO{7^~+bRgy(D!7_fDJc(ATHU9AwD=mYlK(Dncm?By%5^tx{GsKx# zx{_o1A4&drBc~c)F?$hq{rYv4a$9XE+{YX=^&Hqg3P#c|nF1=kP?bG&8_kXIAe*gl zS>;IVjRo!58>*>*GD_eAW}CDUm*nLhFu0z-WbK?^M0a7>EBmEy=Lp+BQyWGAcU2TH zY<-0$MNt?lGG2O{C8b4G$?8_`d;05_m;pdwH}pLCGL-H7;f8x#qy60xvVbu6r!68i z?4_*5`C^aFSDH32L42^tz56~h%z`-vqE@e*{NF+19com~yHA?LTzz~YVjrg7$$5h3DZCI zEa9$3sJOjSVMW0y3Vj9hVTR;NN9S!arJ#pzd&LsjD_mAqG$a9Rhm{N3D*j`EnQ>No{06S=&G*00cw?N7=b{%|EJ)3J#YCU- zz7q;3huHNWUR}(;;XRhTHBkze3_CR_dfZdL zt69Kkm9v;j{k1+Gaz#;;9#j-}B;AM*7oP;AbaWfYp8M9YH_*%h&-S;goo9Yt{_Og} zQTWBlq%?sC)vGzE)%+l>(J$U+WGt=)Jo;tsA-q*dR!>4orhQuL96xbSmf1w|iolCr z1rVdQrr_%lCB2i|cgEkp?;hbx?}ilJEMA+HN|gV|EVb&s^H$!1x2_>!?_{$hm@<9p z5@bzE=mO}L4A~;mf}5B1M=wsaI#BPU!>HLqB27ZYI`q?UHT?@7u+kXCyF0nZpKO_C zd+fi!SRWhDL{Ta_246PPCowb;WqE|nnYt(%6y z#mZkt$-e>V?MnD;wK!g0Y7ZwK@m~r6XOv%Uub*RTpwazf(Klvl9OQSc)sZUhbQuPu zE6<2V#2VK8hRp^Iyz~#nDil7 z{C{4xCW@ue%uL>z!;WjLDTURT7L6_Bo_yMqz}c6T|q=`2eFw3}lVT_OqwZO|V+Lf2>xoc|8wks_ra*;X1zxzx~8g z9n}16+~J<1Q1z;Rn7Mp~u=`jpb@#DWT*6~AZQ!>#Cn!Xa?U96x!#-jPL(n^PH z+K3f}R7KHTx$;;U^#VP)l^ZTea6_^;G}S;)#=5$m2{)|FImF2Y?De=iNdMK#s%gMk zf3%gW`lJ?DL)_yyT&dTUq+QhvwP!<28 z13)}C!PbA^eS&*~5O?HS{6FB9kqGycFROq3?y7B0?~`4z0jY0B{?ZoXxrsOKsr87V zE@WoeP8B!T1#jl4=dPKZ&@#CtX2t+PDAf$u7Yd&eJqWU~ryxreqVs%7=jJ&3>bAan z?_FcKWbss!9J`P2u$lX$kcd{H^RUGz`RZv>*2f1{@K&9${!K-Oqv=?cRA~tGqgaPU z6J;vWH$^TBMPIC zabT4HjtrbAUU_b$I-R&FOqPXb)ZRQMB6pp}TA0@z$J~JVa6uh2xuI>X&SF>KEE_c= z)j(ORQ7+Pw1sHmvuxdw*se$ZVPt1V`){Yrros}Rs$Cv%Qu2!{-?uKTb9;>_Nw6%I} zB~)ev&!Lu2(jBvI5gBsTXzj`f0}a*SK=G;Y_D;#%DpQ@JpjZ4|*Y~^+i|U{sy)NYD zx01zU>*4U+O9Uq#sZ0kUnuhPS(!;jf$H3_U?vZ6T2yp&%=M64NFxrC6ftAUg&|SnG z856?tTy2Rx(Y(uP(<`vEX&*WLBdlQhZauX91ni>}e(g8tE(?|+Y|dgs+}@(`tJFX> zGC`(Gt^-hJSmE&v1Z~YH#PZd z`t?^Y!VYaU;d3t)RTaa=1$RHc-C7+?Ew;;ew3DW~hatD%+?d9r(^*^lu^TyPF)Ack zdmiRZH^t&Bp#^Z*3#-5*adH4z1%K%VmEU129cMV3$YOyu7_tyIKL2>nt2FxQ70wQJ zhBdgfDnnW7sHadk=9ZY4sZmpjd_2T`(HM^VX=+Vi;&WU1VczJBB+*mkx4p^LHr;v* zp1836YmwcZlBs53UQpZmKBsd^%Y!I-Xj_qeQnKUv5y3#w_L`sp!hKVU`sre5Mdj$k zimPt9Eu?UJc&~_!j>K!^dL{ziqFD9*{ROx0GcxH1WTVxi^w@yAD3iZ-93KlEe6!0W zUrUqE#3(|XY(|AmG3<%e=;%j)RQPloR553m&*TJySkvGa064gl^zC(-rt&%T1O#Q; zpTc9d=QEnV;Cd_2Ru_0x!IdoclrX{p;Y07;VPh=cgxIktjK(Mtb}}lL7^s`s^QhPh z;dRCkP2Jp;6bU~kk zEI^x&f1x=1Vvrjv`3M6yp=c`s@pG@s8*fPvnNtFg{h!GEgMx&EG2i!(-s@WdhvExj z6RrAf7tWXz-;X%BBk!yd?|_ilUgP&Y|E{O{Z@~QrC-VOTegSYxwDJ8}mF{u~60PyaVw>a5w}&N4*FkKTW|=R7P! zW#I=1*EPT;7Cs^}_G=@^|H*$fgtNQ_lxqA(!y0)@as08r_EYRZp0?w%-HR1`t>YbP zL)Wq+43?pv@eiWSSHh3@hKrG1^rrO8_1m((3MW>N|3ftnlxqI)3t-?I`PW)JBHdZ@ zD%A7nR{uXyjkz>y!jNz!@x)l#3#dO>S7N>IJF2wpQJK>Cb&00ypi1Cbwj6B@^8*n!>aCh-DBC^ zBivjRzYMXLNjRgjlT4F)$*L!A+jqT#)O9OPF(;-Dztcdc^Xdz}zy!(*njTL04g z#l!l|ld9!vs837{sp#<6iyeGfGy$phc@qGE70$!RhY{#5Dld&y&EWcv^(Ti2?JfZO zekN)DHcw+dr#dR9l_qNzwSX1+Dl$r$L3SS2?mz~d45;*`8SaU%xoP73R#v05?2hD_ zz4|*3$4HHf8eJ_m>v&pZEaiM#YhMGr?zB1hQq})ykt8Gl`7C}O2vWu`d;aScaQS_m zeQ|6+CjEcVdm#OO?jaGlDC_^uivp|{*OA3ut=!{PHj%46U7x%1?pRzY{IxK6#7$fu z5+c~71_sl$`cqU=UMb(!NRyp}{N?ChTDo)d#?^Wo50KI{2#Eq2i#1_rdxRarEMT|e z&!Yb4c+J6Az6`L+%brJ--Bki|Q9%#z4sY4H)fTSPj(P<2n$5ph;wJ-f<@RUM`X zj*Nu=M?vq~j_uDU{kDh_4R~)H9|9wGz)tY<8}L@k>4fu>zv+fR{>5eY;a%zY{OMD3 zFabO^Z5U8~H0ipRKy2P-$5w`SM>4=!m_aM-7wP|D8t;wWt+U%&5TD%^BoxR|2L&j8 zyLfBp>#kFS5H5-f+!(ii)G>dt0|W4=dJo9nbbd^-u^7ULXh~ zD6ib91r9$15&DlxGLrkfI+v&6W$aJ5_52i9rP^irA?CAlfBngk>XZjGLoH8BuBU$E z@on`SwVpTIj@`kknQf}6(p4UwyCDJ~TimgZWwaoMJz7x_z$3oN0rpA_wqdUXVg})V zJhb0K{3BdUF=qh60}*1z z4|l+x6yGE@6g|r_f*khWS;lvg4Gczz$w@_f>p?bMvf!)SAA}w1k^0_K2h_6YIxo;A zJemPCYLB>c@Mt7Yhl}-T#Ai^LDgRA6bAHSt8-kuFLlivj91+|%a)7KKfeg%iuvD<= zZqAn!$*^4YT<%tSXq=aRf$ptD5qHM68EyD-MaeY!{D>?OH=jU zitxJ1G_(1{CBNO^4G;o;y$N^^RhxNq5#T-Gzj%*Xio0Ymo%AVNImswRaBTmx=JYGo zfdS0PajiX42h}3E*Y=6Q8`imhYT2LjrPOA+v0jV2?z`tWD6WPdt(bSGEc>1CaM zmA%Qsi|fv z2+uZRH!JFeBw;6QA^d7l$Y}^+UqC*}DflWEP0>o(a$(1uD=$Rf1s&=O5Coe`A8gHF(KRR+*{$y{!fMdx zVcY}{h#4s!1H~bxWlbax#Id;ml2896Kz-$WkHOdx)4tSmPtHX@aASbZU9_Hp=y-#g z_JJx%59odoVU@Cy=nu6oa}>s>+lpaAhnCm*jnh8gQvN!n;bB2UzQk~Ea_@BNw1MvF z=ak{r8r;35mmujRk&hDKq#$VQU?Bn>!* z!s&jF8J%XLGjCgK9W=%5|w+s(64|Lv6i*q-cjulj; zcUyX7LFu-&d@qG*M0k1Iq_B49GPFM{Q(9LvzBPz^+RkYtdff@k-Y%IPzo*pMafNfO zqBZfk_$+x>Afc88gCN`*?I@0luI}M|F1~2*Sa80dm-Hi`2Ikqd@G?^M}*e zKWA$nYyFCDW6Z?K^O$lnMPCT=$)R3*wQlr9^~v&LVuXwysgU`qR<`aQagCNDuItg* zqG_#qPjrt6%J#NC8@9v(E&yF|LfAPel-T^(oAwUZ=yq`7zv6?=sbOw;R*aX2p*r2f zqA~H&qlLQcu>uZ7C%qq*3u`YI?Jcq^e)3-#DvG>h#A5jhoHSyh1n-&mXlwH&c+asv z-m{6)Xt(DZO>xH;9)_X-b8t2$KaxL1;Tv zc7aQGoNuW+8bpvB%yB}JF7O&lw{N>!AXcpmKKkYfc|*QD+=_aL(kZQ$QxraS@raGX zTitje$>M%H-VRB3EUQ{F$wLOVqgT z3aeW`#HCi_acCPGX@Q3=0>oES2;~Ma3L$=fnAi=XF@AxH@Ysn`N1F}9fR%dO@T)sh zcd`}Zh3-$Sq=#5^+-|7bINIKtd|)z}lTm*VaeSl=ee^!M+0cjUaE!rXWvHq|5BhD2 zskvuMs}M+wL$AxQ^>7+iA36?i5W1kteyacJJgi8^9|9Efth_u*hs7uC4UWy~ABrhq zkL$X(jvT0#+I~kra&nHr*#jKBG077qQA*(LN1{qc4bCu8B^n})0xZIpnmp+cnspOP zClc5pajXY=ea3$K`?s(*?GV};bbR#ih(bnbg;pjQK{pOPM3y?H>W#Rd%PMz6MFy(3 zH<#YU%CAjUsje@LD4ZNRE1@@=M@(Mr2$uECUxf8)`yB$b;;hl?Ic#H~JN3$6x21}I zRNtuk{f(2#+O4ewnM=z~czK!uZeU%{2*s{^%GDyUu3mqvYa3UCwfafIS8xmi0nd{P zqAJ=eb!or6+_!EHEp&Izm{AgWA)c#FZT;NSX{hbpOaqUihg6O(lSXD7UYdNB^ zd!%*bU_xhRi^$y6k}UP1$2;r}ihVHwTBmQ2*tp`g(@WMJ+u2=FS|OQ7xI&%4z~C63 z<>w#Jb~}V-9I?XSP>YGVKqu1X{r$wA|G!a7+GHt75h_I`RJIU` zc6+jCOR}4avCP=o5J?fTXUWctZDbjg?E5x@F{8*b#!#3UjF~wf_wQc1pYQXWKhF8* zJm>lSo33lF>-y}k?cLAdtt#r7J3FJg1N$~TXtTe*1Y?okZeJy3QsvBZ50z{VF1aG1YgY@o~m{bYK+Q_idE=fu}`Yy0lCblMoBMC|dXpxLLZ z6}i;rQ^r3&<{KY_saJ-CeoZc_)DQWDN#%M#sN%C+qYTmEC3=%Ww7V?lZR#Jc79{+z}3KmDK`K|hmY5~ zlYmrXG!N8c>pb8B`_8opYQUg-c;78@OmuAk>Y-cH?YOkO&S@*XR{iNC)Q&Tr%VFzQP)yY&9>`F87?SiONtpz`Jeh6mt8U0co*8o&d9wGzj>N*PAWD zXOBX$)&Iqo@b63qV?Dql#`ZcO1LFVgODDH+s1{YB6|+Y))#ZKeqTGxaj(&e&9)?KJ z`JPhb+R$Jjp!_{K?0ynLctOMPk|kbOKyk?*q3;YujKY=w04KOWpn}X#$&*Mhr+Ri^#d7CZl=hZ<@eY7nYi8Mjo{;?aul$7_{slY$g>+N)h)=EcX&tgV@z*PZm*+ZaSmeAn|89xyBP z`cJ?`$*4_ftE#1k8W(;CM4w;#aOdUZ?UL1?43#wV9aXM?!Xr7QW!6ZfAY)uy#NKj4IXK5Xg9JWvnd>U;T#ah{SCNt$$V%E zg|0ocPXd@YLsvF#tq(U~HFr*%S*})|)!PCb4{QOB z6po^SxhZc9gm60Xx%rg*i>LqGy# zQ1u1gPyvy2ncv8w8+Si^Qgr`R2gSSDll-9?fLVTfgCGrl>O7RQNE^%sKBILWJJ6-P zJAZ47Cp(=HZiz9?AMdp9doq-68wcd#a$xTS9CoUAa6SwB8Ma|UUV%kfU;d#RE*;|- z>nvz`u5&={0W1B6wf$&3(80%VZAyWM$H>uwyS%JO6B@efH`vmTtH`!|aA4s0uL;x? z{?X5Az)b+0b4xid5f|5t4~VKh^}Fi(=pVjjb}-}oC)W@%T%Pi(m3elkAATAn6bYZ3 z-Gw~oZ~pS{p$PirUNAfEe4SkfL`3rcP_ciF*44)C4|xF|01{&Ut&rYIh+&%rv@D2~ z`5#2gz%u#&sf$R4FXhP44@5rz0%lf17~GN%RFiqkT!{bA^Ka()~yeIrJ1*D z?(aHfk3KN7<#)hr`5kh6LhzraZP$J2ybdz|A`$W$gHA-~tPjvnSn56T0HD3b7C@~4 z%4$7$5br)xchzTM%;U_z+ieq~=)* zi^LW3h_^qn%CTh|8wceDj_0=5H5Io8uSJB(Au$1?A_HU=<2d%{0x6AxWKlsL0Z0thb%<`5# z1~ysM-Pxugj&lz^XE}ePCzq{0AbCz~-F6S?ZOMSl#oMb-K=C;;|3T)Sj+2&YE}m^Z z5c-F61c=?Y{(JXGO?MEX@Ep(Y578#Y9;o3ydi%tR3*;p^lr8zaZAI}eWdEH^Ssg$o z^wg}JjqZL4+lTyT9tR4))l${C-R+_;!DrC40K4oQ!1$UWCXOdsSp3JsZ1oMx=Ev)m zwa=;AA$s9*^DWy(^3L=h0FShVFDbD7Zg$|`{X{sfBW&`*W3`AUbN5%U5rb-b&7n zD^AhRfW`kW>%m~1GtyS+Ovbw=}HJkWC=*k&T@mG5Kwc1jI zc}wz;px+F!|2+8jV{E+Yt7lc#--QnR!u0Iz^#8y6g(H?B=vpU(>QBTIdqJ!u!JqDN z=)#dQp(r+z*)1{+44nXbjliDFs-)^ic3)w!bcgNQZ`>ahg z3w5H=i#q-SY~V6QAW+Ni@9Eb7*)xTZ1c;%N##fJ)15hx_HQA0~A^Q zs8<+2YC65&uDq=5#}<$_5zNvHqEnPf*EH0!dp_^zJQ{ZR*p8tRnX8cpx=pbFH)_{( zW0oc)$XefetwOy3ul=YK--ZP>dJ_|CryHN~#t7)U9{~!{UKG0gh#cJqHH>xajPHon zNoXIdWZj?$D`wlEC8j^9La&4w2GhHhJ^Mj0%&q2IapV7MYU0MnTXE>-$Eg!2kK{AV z4e8Pg4R5fUB`xk5U-(Uv|0CS_?&7H^_#a*EMEqcr%>M`A(**- z{eabMn2j&|>)z`(vIh0NB5(*;0H9Rf0aEKK1|W;mMP9QZ3_j#o*4uX#V=bD~_v>A! zZxzbaA7EJY^?J?Ohgr>~mPwWuiPo_X&t0`YpL5M>`(BpZRT(!t? zY~7-jbl}I;RYXeQchEoiF26wK=s>&3G_cB1Gx#p%F zD7L&=OBGG2s;*~#a!(u|Kt)-Vuo_zz`qno;Jz6~PbPe7wE^_fJuXE4&KuWC-n&DH8 zb`yf^O;nsUrUlleU*-&^qdf5BtnRhmVn5pVq0zNhXpa(QK565eqKj%XE>4vx!=Fko zuKko)qyq%mJ(#(bwb7Y!NIe>eRb^n$PA&aV#`>f|QUwNW{hO`2(lm2#3RII8;o07a z=SMNcr9r%lfM5nYV-{Xo4|@R3-Q2{jE&V_X2kl2TqikNB$w|| zz#qH;VnHSQFIrYla==E#Xyug2Jpp#5bDF4YACS8NQyAI~ddD#H7e1hB2_hA%HSPCI<;Y+Nl1pg4 zy2;F>JNcHw)+ufE^78W289{-65NV{9-XYEJ+xo>WIRQrj&sdc&@gD7)s$(XaFp!a@ z07*nEt65qEzh4%A7MvWbuVJy>@+j)Y+hdET5pDnqAx|?e&4P1OK&T(5yAAbwz(Ghz zu@x=c9YbKF1Ky>rSahs*w%9m~MfvqM>xy;iDI2fg3s{(v=lDr$Ri8E=*nNFLMH#unymu#l6Yy(>2$B-hYx% z+m{yn{l&nC&o`+W8foe{v8Ti#GT&puhv|3yHqD{A3NgVf{Asy{waoI7AoSvfNOr8? z!PHZ414*e~6y6%b&8BaADf2IRr@TFNe^@VpC(x~&RrkHVN=Wl%a`~Jct9Hxn3Ybj| zx3M7m8z+xT2ETvYfklPdrw<`B7%3*AnE6ys_x{^sz6*1*5i1?k*4O44>QhHZdC1DrTj3~f4f`o^o=NC zMe=O-{6lMUHFZ0bT&g~wmo;5A5fo>Zs~2gm%zW{Z%I{k&om3t4{V`_Lsgt?m@Dc8r z-slU_&}1wDh9flekoaeDHQukCAv`$Y5?_niFFTgdLVMjQoqNF4b84@+sA@2I*z6^B za~`PDWQaa=(c6PC>j#t{1_zZfctlVdAWY@w`CZ`-OwdxF0QC4MmMkv<4pqr9e;%E} z`2hjYqSx~%?69k|$i;EeEd8fl8st!(s>5caai)4SpHsn-YyWqCo= zvc_vxzq*dt>>2%jds0Tz^-uKW9WtV2@TEZ7O&F}kx<&lVW@}PgJ){n0aRss@!6_yv zI!V;^(pWc#S^2tB^$Bc+O&`(N;0^fnYSPypQ^+?2i{e9~ptI6o9ah#u{wybqRq0!hHkMkPOXp;J7)?QJfAL?4;<}^0hO~Zat z%j)->G%b4Xt9-ttEcK#GX6=GKsqW%bSNS-tv)CT3JmDQnXXYq*WOsMNb+ZyR<$Y2D z$pxMt&=fu@nZ(>PciuzC!#H)7`-NvJq;i%LJ@s1CCN%0Gks3jw0wU|(QUkq5HVZvD zXamRFq=!edvHV~SnXy>;+;ZZT?}V4m`ZJ7oVOXsk z+pE8_0NR3MkMnHTkIv$i4_Y5?n;or*yPDYUG69TmkBL`VUR9rdlyxqUgz;Rg?u#1t z!|Rk7CcdT0jl3}LkU9p&YO#jq)ZOEgbsAl7rkN#=ru7%Of9|xHt=njNM@`aLUp=Ut zqB?y|bNxBk$&LC;8{4EwDza)@MwaHu9^T5x2Qj}6J?2KfTx=4tqip(O{S^hT4Z;oZ=a)il*f`r?xGtnI@m z)(^%y<=XSUH{q8FsD}sUHEdGOH~ULld)9U@e%AXsi_m1fo59-7V)B*C=O6W=Z&yDK zQ0DcL-g1IX0?@goxza#z%ls_$&rAeXZsV07d2g2ohVp$>&`Lc-mEmNrD7a_u6g0)= zD*Ey(!pUn}qzm;tgGpb)-~$pmk`xPzkAAgUu>ZQnp`cTld!-v)5$|lGN$Fx_R`}+` zt3%Yul#Z{p5Oh;1@49zRN+@ju@_LE~3A#twqG8=gzSipA^M4YZ<@s1PQs01itfsC% zd^Z}`ovv)QSnstY8iH<};X%4Gsa?J&WMt?KN88u-fJDA4wC`Z@x>8#;EHIz$y>scs z>^0tJ(HOBF{~zv4tCsX`3b&Xj&$rao#a?9_-{RUfxI=cBxZfp0Fr@d5D@$e3^G%cP zF%8w{YXzOMQ3r*KYx~pe$y^vUNm5>zv#(9h_lM@M*f<^m)#p^>SZl#y9w=k65{YbmJlcZ6&Gy{#z)){2tSG5q()_ zNvnvyVnA@9?HB<&h-a_1J2(li|1jQ1eygyt=Pp{hOx|BRv#Pb%A|2S4t*gqtD>SEwgvk`?OO{y59!fu2#g;s$0X#qm7qTCTASbpeKUd*842lEQ z98e&%6Hif+;keO!V?-%>rb9ZVYLW*@_&x{oI*Gpdj`%JwTC6tcP?qx34i{Hg`-y>4 zFCnqqSONady=_@wRgZp1Zv@KOscXmoJdJeCT&3c9qYfg4t6 zl$?_PallRhA9gcjc=pS@kQ|}?98K1%uw~WcaBm02o*({ z$D4Mlw9eoqt5$Akvf%Pg9j~eguP#WTp>sl4qOFw2Clir2f-`%8%5e*wuqSM#3c&uP z^)5ZXqz~Ti`;gmD?*S)=`7HC4j4y<5?~FsNU99Ecj9_f1t0>HAB-yeGwdezX2Unj9 zZw(vu4H>4i??Jv?u!10mR==oY<>ntvD<}-B1z3R7lO7FMQ|)^ilYBf19DpK^UtDjx zQ$achm12JHCR}fNzkTF>KNBnhT!AV z@yA={O{ZqG_n=B`&^XDdivwqC!K}jiRTTY$RitJQmkNwuuAWU@vi=IBILD6w(G?rw z4FrgrJum$!o;ad)B$urgFh(vL4crnPst^LoGv2sH_YmC!NDf+Xke7sABZ-;TQ$(`; ztp_1`-V=nWfhij8%SPBCw?h!*uL+9j(rK%Yw*+No^o5H63MD_BXCyJt6P`lZnCCNA zb+-&NH7lr~wOq7BP&MT4A{+5TbaSCWw&Yj6Z>+!##U7sAkao&BXIAOF{=3J2c{pbFCX59@OxjOY11o zY}L^GTyJl2Oaqp%1ay3(&>X8Ksg!gxC#kWiqTXhx=-@FIi{9MDAx~+T( z3OI~!xyYXEd;+Gp1IrU!CJEN#tNW^PD?1axzcdK_1Ue%7A~V}!lW>CFgD-2p_F`Kk z5teks?CC)Ry_K#Vytkb&>+IZdC@ZA$mkqT(Q$@WfDV zv=L_2%;r&N-378>zUPaPtqIS?0(~@^8jjWYc}XpV=;e#Z(*S2S&VYC6juC zlc#UTVMCk^3LM8=1`9sIQwsPXn5xfR&AR_ zH^9h`gCq<+ek4BxDE>V!{?k`XLm1R#$b!=ynDS4JsoM#&`|Yp@TRAfu*h^nyx3!;d zZx_R$>yGQo?%1gle|#eKx!fW*4%-yf6O(Ib$STa9rvpd`l%|T`oDW*?>T#u;ARft> z)z_V|{L~j%6j{>d^8WGH7K{kkx|f(WI4hHWX*OXL8eQ=(gZ>;8Jjjg~ml5!#N9ndj z^kKW;+fAWmUak85wMuaEVhvrazg3F(Lzz}p9{*C%4$6=>h(5scj&KwH6gqH(n7Z0| zlo8XGxIDI9_!Xk|L9fs?eQQ|HeD61KF254coSLxE&;F)sue{+-3=(>Y zVP9ZEQZhB5UVyv}Uw4ikeF;KQ9bX1)k7Y}4)6nHCTGS5PIniHS2zNMdeZXVDh(pz~ zF>!OIe@vx{tx)s2_70vHHau7I37}2J!=Xe$f(_O&D zM*urs|0?MPE3m>@z4>Ry%Y3|m<}NjEQ`=m|$XZX2kr2M7qX+_<2zk3o4P(_Tr+w>u z=9wEUKpd}34!8(M)&1P`gI5RV6;|0SA#YXfb88W)r}VVQHK^ii$9Q}FN2{u0x-pfF z4J>MMpt^lxnPavOnUjnpiWAo79G+$OyX5*S3!=3bB8_v*dxugdZET7A`+5s610*To z);Us=PV6bm_X8x`>%8J(zs4!#a2BJs&goup{TNZ`Sge$O zg{LL;G0N+5?~-&$-RXIJ2~^Xc((ggB#7Y(SZ|nK#Y@HLECUM=W_bK^4*Nir=x_7S@ z6HFc1TQS<#EapxFEL{1}pF1nVBY^cb<9tr&Y75p*{J?jT5Eu-Ue+%Q)O&UeJF;hVb z59wR#w|$SMQk_~}$2O5L6ru77H#Q3gb!qQsj&IV^WEEX8U4s%jnOq~r2Dp_uQ8c^^ zV_V^DMBjdeX;YWC<3Q_|`+=e6wO!GHb4=fb5qrMx1Q2x2aD0xs-X3$pLDX^p zmK{P!IqHH5hLOvNLI+>2&7lg5lS3Wv-@mcGTQJ*oxL~iusTeAIKCsxjgX8La+D@)7!3#FQV%yX=&)}N3oJVBmHVSuL+XhUVoU*C= zRC^{o(VSf=oM9xjMQ&UofI;N%g%d)7k5dJhnm@G;vK&mcR@G+3JXc6ba0f!|(>^=K zavjJt+4cpPE_s}4f3z9_U_S0|a%<1FRo`kar??~cBEUt`1RH?(ad zm(26`1N`vIE%+^aaqsmu zfbN}^N`D-22WTRYsIQU4ETdJ)D?LMXS9R#ZxBgf%0O$4{aA%=eqPr(gf)xX}$^JWc z=zr&!rQ1@CC6*00%gW0BgQEv2Y&R(p)5}{KT2eAH>KSC3d)Lym%@v?%lb8trh8e)% zb@o`-I=CYD{YF$i~g@-E@4=3<7fCczThWCDF z{J$LgpdAJhC+riq!~v(a=tx&CbH{%Buc|5Ue`m#?9_X`gQVsa@FIm9CVwbK(l}qHD z^uGiLCk&6PTW=AF8g|%CZZ{_S_5PB{U3Vz$X<4`eKEudL{W zg#{*Ewdr>oTpb4nF$yx*V%8wIUyB#kmVuf=1PGb*1Xyo7Iwn2qTgHFmdoO)>1G0Bw zCqlrC^PgtM#(4|<7Kh-2^KL(kC~Z~Z+vfysZwcblt8{=4UV+94DW{=22) z5deY0n3Cw~9j3o!V1_i-OBj|vZayAx@b6n1ghaJB{p)VwV2^R_;qmD$hu7Qx(C+X9 z36%i{n21mPwTt>&EChB;-wp#kNY;PKgg}&H4?8%*o&N8O{$GBG(>+fuEG(9X6Kp%u zgPEst(|iCob^An*@JSm(4iHlpTb&*(3w%aCI5MCyxS%H&K&c%n`xD!s_2v!QH)$2- z+SK$LZNBzFsaw}X{m0IUY~^qoZF4m(Iq2w87g;umw@a7Z^28BtqwLBBf#PDD1);bF7( zLr!VL18~PNh4s0n=mnZn8Z_swFN3nE`XG~gfq=&GZuVnm>J(&T^gzg5k{3JX`E9$6 z!olg-XiZc!xnj^D)-6eqd-~DL>}8%H_cxmi+5}bB3hx~|q?u8#p8odjQS=f?rnWd3 zJ}@{qopD_(etEjH73$ocCs87L`ydz_ZNTc0%xx$J%=l6Lx1}4IKb!cLzqT2v%)gT{ zGX<&wLlUbA2p;tJ1*T((Xy1uAkobTn+XFt#sH+Nyw-VPnfN^}m zq`L%d+T$63vRY_2GH3(gd!X{ERaJ?1EknsyRKA}pM?dj2U{Q{Puqd5MhT63%MVbq z#fvb8*x-62FL>(q*F>nT-uy#u`7QCok7r;RS#SGeL3Y=_V^U}mi!1teJ45S5O2pT@ z^`P|FA^L(BRZc-j89lQ5MwCOmL1i2$R)e^n>G8d|Gcj7wF4%@yuA8LLKX23(BLb;2 z;zhtel^;T=0L0lVWc0cah@e$=2iT*IexicK0U68VtyqMdj8)~U5x?OAljS~6zM9VM ze4@ur=)`+ZsL7YkKkV4D8}0m&u=Qx5`U#MH@*)Al_W0v!O{<`^SzAF|;kY9VUl3%K zhUG65BpO4_W>gBdH zbHD5=?~!3sXhHAG5a$6ymF1TUO)ndOiS3ZGPYPUp;zWYYX;$rAFjwFleVoZ%Bcsc) zpb?hVEAG7I8DLP=#?ge)DOnXz4-8N1+HV7nr&Ls@Oyh%zd%ZD!9Pav1m z+y~C^^US-kEZipg=0nCRryDula7{RH{GRqKQ0Da~HYbeJgzox18A5ez zxD~M#lBe!+!iS8mkNF<+L3!CIyN}qBSLD~f)5Y&IHYij|_YN`d4?_TUbZg-GsGE3R z--!PH>0@u-eP0s&I2U?S)f~S+eNp!CyB2ADvsFV`8U(Ih7Y(dPdDL>q2yrugrTuU} z6-yj(h>qJXrTC^-gVDwG-z4=mEM(AU6;xVnmRO^pI(9)IelbQoz74oefSo~j;K#Vv zbelwUMJHsU-kOlQUJ_VbfucPz#ff=Z3lw8@NZo#!dfYQ|Z~kz>(APlCu8%OvJhgXv z7pC{2&6is0Sa-UnZO~n67Nfjm*0OSbxJZ=`dG~cp)@#+%IQug!qEw=y8w!2*`rErf z&C{7plMQS4-k!0l(e<%Z7AUaD3QY;rcTkxOmnidCI@9M-;beV1Cmfp9mV}w$arN)J zyyGVDh^3{A&-$fc>o)7?xcX4ax@PK1{&eZk27Hw-|4x1kg+|SI+}|wpl8HX-F(ASU zv}d5J8(pJ=F`E)Vr4z#`!_9yjK{^0aFudYC$t{3Com}PY$+s6na%3%~vSP`YiHV*G zWy`q>B>iTXsQWGwVb_LlzP`$^Jz{mU^zrm9l2`X$hMEOSeSj`^r{L~rea1T`dcTKr z8Otf}`zaoB`e*Tax=m17&j~blEfe?nE%cPGzV9G5onoNDyuw^#_^uCb+^sE2cVtvu zVObBjWeXf?{;}@Ybu8pk__lp~$nhs56Dhz1bfXJodTRD6PXtI9Xey4`6g9Bz5k5b{ ze4!&dGL2qh2PSnD@iM8o}8 zhtFvRt*;BQPSeqZ7uCah#Xp`47QQ>Q;JVrAxXr#=SazoTAFw+Ww<;`%?9~ zu|Cu<0!N8LG4TWGADsu8I~L@sCLV=}-awNWn%crGlt^t>c6zz-tkv|-CW)n89}@6$ z4N&PKIpZw7jTf2By19l(;7~v=f%PqRB7WP*CtVOEC(HO~Dqfyx3?{3INs=-_)#c$3 zAqJ&9aag|5bu=4bF4W7qz}*RB-!nQRf7MB+e*xjQQ zS)_k=B6V`9@9-W-<7GV7g8O*y?&vSMm>v9nySd_^J>Aw^xw`*m@C=}<$I`}dGZi1H z1tif(iqJi$W#)3Rru0~B0Y$J>tUF8H9BeuZpiv%FwnFt=!b;vErOe%T(XPEdfL*&F z`mm9GIcqq!K+{1ut+GhtX=kEkm50Mb{oyalN9jQ=MCv2v>fqz`o(=lf z>dvB|(hT)x$ylc7k8SSi>GOy0wieVD;oat5arDC~5_;e~)yj?YrE}S~5kx19+=Tbn zPAl&9z3 zB{{7c!*q&o`-uc|ZBw_A&ql0LtnJU$3`#%D4w@WYJS$bNK)IS)874v(mss9hnlRPnkE-V6(Mf#m9gs-NM3cQ+zK^@TktBPmDx89d774d!X~h(S$45J|@0sYQ(n zSJz4B%kwnYSFMmJpAA_YOdnuVn2KI7>+2%^rfmlBAvIRpI~%@1u`a7hk`Z=;jnQZkBQw!g}JGSScz37+xEZLL~`O}rN*6?r8f$xpl z+j$OnTu=2rSgH|pQwo0_0f*Pvo}pB;g5W!A(E}UW&k|N{6gw3CbUm+i8GcGU2(bH` z7=D;pwAiB1?Tzva)ap{IBU=rhs8TVwb-RciG?ACSbkBNw#?d>t`-|h%ixZkP%%lg{ z8&h6A{TA?f7Enj7=qg?yCEOMcU3Jn4CZq?!UeUQ;d-5BEbd!1wGBuY7C~R*=lb=W` ztbpOK_)RZWwOKSVGdP-|vCjnEo5D)I|hMfO{6ZE}-e zvxy`C2%yuEbxdu;#rI%5FMaUg56}$H71-wGwNm?jyz9pTS;n*1%rB(P$Hf`FM3Vtq zeK^4jLGOInu%oDP&DNlx#OrDH;$Xu>6(L}-KIaw4e4P@WT&Rf)d)NZl-#P5cA&AHgA7O?Gn7nJ7^}I4L!j#^hp0}ARn@2TxiP&5Y|f&I?|heeZ7C&Ko2oP# z!A)q`>nAv{>aQg=j0a&VonSwH?xku}Cak?>hig5T1%khNzEuG<0juWn?d5?M#ch;N zUQ4so;9pr#=iZD^e+FGg3XHF|KYNX4_4oG2#v}~2R4urRvAGgXy0e29)}KA&IZ`yli|l0D}A!`Kh(9LEhk zQAgvRT0YD<)qhEg{O~PXrTI&(?2zZDW4KQ%uov}Qauu0l>W_qVuhjRBs?Vw{(KYA` zzD{j$f2vACug2{3X3yfNLzr-IJ5%%W)hEnJ9a$HKp3jmku#j79P#~Yg$yeij(>bjU zXi!2uIg1*|!ahaZC&|5Mv3*WFk4%4Er_4FA4i1Fx71ge3Bi^EW$T_73`aJq>if|ix zE}N4#H~IEj>e`Zw%{TFF{Bt^>ORjX^3Msyy;}QDu z20Hkd_wp$zq1gIQ8%%5&TOo}3^HQoVG5gk4?avAU2966Or4LZ#sNI5kKQjb+L0k#q z{S9r3lzxh{hCKAvFVoaLr0gLc(Yg_WapBA4dy&D*S%q}rr!yV4ws32bn4O-_$D4?I z5#n|x4X##eQs1N)rHfN)utrw~eXXKl3}Obj6sQ|E(M{;g$pifaovC;$D)i*xPK@#Z zqIvrWx09{mo=3yfo+NeL<_rX_c4dV848~>6_nspY2EVfz3*qS==Ev(dYx+PRT^I7- zoaEyaJM%4QLO+N)j0mXoiwvEzc}$$OgqZQACVT5?zaH*XwEvQJ)gW#{Vi=~~4N zitbd5lOT2WjW3R(XAI2ui`dnw1l6b$7$~!)%&}Ivd6&A?7+;)MkE7QD|Cn{;Y35Z` zy`pCdAJ zi1JPqi}v17hCoIceN>Ab!)M0iZ-ggISZy;ZMxd*V8;lWQ4;qHiSKTlrrBZ?-mg+}g zctOp^c@1lx>v#IwBlUgoeGX;~3nUS$20~|{Jr!oCu)1uCQ_Fw{DNZ3^F@kS~<#TL4 zP;L73J3aW2sq;kIv&!aIE4fu1dbvJ*eT{AH zUoN*?eI7%mnY>L{xL|UJdAI7qgD%s9{m-vG6V%)H@k7zRzgkV(1Hrfe8LKYdD;pGo zW-LME3r&k}$GsL9B`OBXhJ4?HWbOkO< zx63;O1wMVkj$_{o93AH2GEW_M%drm}hAGN_)Qaawc$!e(&=Iu6$2Is>rfR|mP7>3t z(%UEyaAf_!@FCr>Mdz@iX}1Nr`b0cJidDNvrfSc7y2t!6Dn2_C)em7!f~^^5{p#K| z39_i?9PSoYC(rgKt&d{Vtp}aDkY<~$Jm!@TZl%)K97NcZ?4*fD1e7(2uaOvrD>^@Np_X42_!55{UBjTrR)S82O09e?UY1!3YWo0ur z!Xr}%zPzYv&+1xI?t<>uF#rrzPD3=}rVdp9!Z(%}UvH`?r~&@Oy-P}3nhNNKrmZR<~Gu@E3? z4(i-q$tc{#hJWsyl-H}CB(qwLz1P7}WD~snQYum`@WtIqGq_PNWt3{;>dRirfCgP- zd$I}y58YLV*ikVq_3Rk+%u9FYRfZUP8Dd|-_m`2|Z)eiF)ZC|@qFOp^5+JB2D^ITT zid@2LGSyR)$VN?vLDgjVHwF8s2{`CZT)5F1-$8On49e2K_%36yurPXh3w#0_0jI!&1GICMy4eR*(%75 zu~x?ogAbVN$wI$h<%D0s!|`UppQD#%T=FtqSf*Jx#tlK4H1}jgiI8EAQ0h9u*3vDw07T z)`jk!{E{jD?aTDEnT^pa_kP*u8ngm|-N=HhOd}P!F@H82q{MTDA2!+v*6Ux5u8$h= zaEdq0*vtaeTK0)cAs+SUoya)dl|9Ja1*u~A+bu#~$Fgay+st{?>KwnzG|#WTGSRD; zd~iquuMtf-g5zF@(ElliidAztSP&6PbqPo*+!rnuekdsN+PEq{?7at)B;kk2_45hx(300|q zN<<@O=T$lX*wwx8p-8=X%zY+#XGgFp^)3CzAJc z&5_LhlO>;>@%+O-eLZ|59>t&t1Ew*pk)43clpwIM425 z3u0M=Vs?z1R{Lud*rVXrvg2pdvrpY1eXV31pL6pK>UM6?5mZ0c(ybA&AQdeWBx@7C znTtdHY%T5o-l%UQp&Uo$Hw0f^g?V36fSO__rJIO~i`Cp~m}`g6 z&O+sz>9~gsFXYWXy3>E$-Z)1eUm#?PzP_6uG8V~s+G%Xpdj+GITV3tGM*U2ItyWmp z^kS+b3)8JUs)9Qoj`0QjIEseQHwBr2G(``==!8YY}^Q2;RZBXe4@T*tGzs| za@h)Jzq;-_XEr9TofR1YAW~<&$&WA-j-m(8FkI|r+u`v(79kX;&zSQX&@Z`N;$rY3 ztGkc-7c6T7=PS+n1M^i|?(Pa)G(i}L1o|~M6kv`D!poL$O$95Rkm7|~AyFqHRrU^)-Qy^lm#HSn;v8ss^&WBh@|QQFzw~GE zYiyK8C-wU%-BI^l{T^mou?QJ>I>uS_;Kpz{8~%-KH-CP6)+%LvxL1TS_sJr{;3Dyg z*Xp^}3RG+0lx9s6L*JIM%V2XNA#lPYWccQ0=GDkoU!HY9Ms{gS8W0eEMct}u-@65< zzj!@u6~`?F9#>l^n>2Wa40CO#ye3XQi_lupK>7OOJgz4?r86@8XNb90>J_P>Cdx!} z_bbs%0dPRHP18tTWY+zfY8BE}^c{r?twf}T==Q#3AbcNOqNE!Y;a{t)`&`U$P_8(m zd1am5^t%>1eo@T3`(%mJtNF}#C_2HhGfE7R{TM8$N%B1x2R&}LBA60z@1vwPgTpTA zxQhh$@lT&OYtiA^H^=X~s5YWBwJn6Wsb%F!kSaW!%!P7_9Cc>E7YnzI)U+B@wZikr zj-#g<=M}9l^#WC?vsV^W zCAU3yY&KjX`lPAF0Xvh=th$rmNb5g#zhZg7hZyz?8zL7RX1Xw5cPWIvSsSP-mw9ko zwXEMf6Ng@aRq_d6L(o6rubQae)2m81{MT3o5!>-+X0F`WnqF!Q^n35DcB)V<-ovZZ zxs4H;E35L?_(CD+#5;(5W#}@K?t56PNOk^Z-`Vp#UgSFp5=5d?ip}bL1-AcXy#A2& z*&!0_16s`8w$fX~u2O^RIJiq_Y6L7+Q_T%y8f%~LiDxUMl>`KQOs$)LZE85Yw3#WO zV!iF)Lcc}I(hM(pZRGmoILDo4H>K=Y z!Cdk2^b9c!89u*hbz{{F-hVaBi=5S?a5KYJD;+*e-fu^#(q$RI=64vZM;`vkAjC?J zIxVGH_#aqg=bjYWd8AJsn6qb+pR!qvzGD;7IRK_Z$L8(k!^67Os`e)RH#-$q5X1{# z4QZBvf$cH(F1#mk-9fy2@PP3vI%o*C?HZiVz4ReHI<47nX?gkq$c8Uw%MrLonRA_) z210Z&_mnOME|Ty%2?wM|i?jOZ%0-I4ZSqc}CF;xCc{O?DV)=lfv6({taCV^N=n?;c zkn2bD^_AI8m#`aRpO%=DnIS=@{ktaT`m?&F$hsev<)rmg;)@-!Z439=RTj&kS7R(Z z;nI|mgQ*8=GaV+Ut_Rfl&6H^t*&td|0&1t9zsC2Kfh`gygZq?@M|3Q;iPADpgFPUJ zk-1K(Bh2mPQ77wMY+nMdK|pwn{90PnSoj!h zTwK(4T;pe3Q}q7f!tLhh{Ka3!G(WydG%Ej0<2*Eq-;nNF5kU`_zZda9Qe)v~Q`cr8 z;v7hdc`azY&42UP(imkkbFMv*MJLM#kRN)*y=N=*fPDk_^+F#@_1l-$PmiV|dWdPVgcVSXSLp|X36DCnWtl$$WGS;cyEJ&((uQkCbhMwiLs;3>+m&P zmWtbrPO`>|3yajYtL_~_o6UEd%tfe|nl|=jDc$kbDjYq7lR6)P50xYlx3MbCY~>p^ zhvg#0RMpFl`=r;5@=~smzM6iAn&FyqVo~o03SXbYX$(OXMx#UPSR27X*1G&TPJ43i z>_2XUUvoVtQ5bp|tD?bRm>eDZR-%i7%AhPke|yrcN@HGCN&Uz?8~Ot1%G`+R6Rxpi<`c zKSNAbOa3?|9&N;&FBZDD3t5Sbg$Pa!PUcVgTJo*Dae^Sy7k~umqUsHT3g&4|2esSd0I}0b_b{#dZ_g|sokG< z1mABgsy}qr8y$JUN3V_JPZ`{Y9a~MS%)yR|KgEtZAOg|)BMjod5+Nl1xT{`Bv4^Gl z*T;PB9`1zSLw+7}P&m%e8sZfd*ipKnHX3ZM5w`r6@xpX&BgHD(8t#U9wrhx!(D8zr8dY z2XS}ZW_fy2pytKW77#+D?fmC8oC{zPurERXni=@G{I7S=QpHxiyu-4*Ok-iju5r`LNf{;oU9L9X_Pm z0y4{gKCCy<<66{d>+^EI3#~4xazH!??viAc&tN&uXt^d}=g_wkdG6LnXu%G1nMJu? z)y^)fht|Xx|Ghvs@b{M(*!$kApKFGS^DM0Xylv#*1rGODw~_j2etNZvSC_xM6IX&h z*Khxyf6n7cL-jA17eF}`m1BMib6*dMiaT9wCfDftc6M{vscT#saenDgr5;h(?_a+J zGx$dCh9p)eSi5>M1}31TkG74NU*vEvy5oZ0U^k5vEysC2AeK$f`1;b(1C>Aj^%+-R zH|mfuNwEeOKNs0nm2^fOYi!}zXXo(oDW!(H)N*skWj4uW)xBCAdNFmcZu-Q)*UJS2GKGhbj^os&qY zkeW#SZy)&VI0vMG62IO|LL{3Q1Wjx?cQ$cV4ft=8{C0(Q`|JvxxkySv_ZTJ~4Gijo zuJ`924CQ$+Y*&6Syu5QyM&nF3@Gv86AwZC!aK-Ito#_jJF9}K)Yl4B~9ry3i=a_xwP)OMV`i;C!xka(lw%aGrg?H{+9f!6S7_!xh^`a*pdHXotv5f4}-$ zx7+}ajJvb4&n`J;k8Id~+*u0Tx$NfuHK55^Y|~vaOcn&1e0K`aWY^}?7~cH^^zOn8 zC#E+6E7+!cqU%jzGD1$tQa zoSNbRzU5^FvFOq8LXz>&>jHCD#Q!5 zbU!Bo6CR)S&h5a>ChZri?+YbWulK#71doiy8lcmZ?|x&sF)^=k#aH2!#WPkQEd)O> zfkPmsy=Ch$C(aF~7u??IvfP}wIsGic=RY|GVyf4k<=n9Mno{6xXLQ$XW#L%2^+wx6 z;Awn`4syFf1yb72O;c5dr_d+r4h3(&nyDZ7R>kd_EAMF5)#-&WoP$RT=(#CN+3P(U z-X3&&nKzkZ^WC2x^AN7LVc}S-xy_i->}Xi`y|hC)CZeKpB7 z=G=TSBm_}yy-;R!%9b}>rg-36aCX=&ai&~VJuQU8RFoVNfRhASxoW_=HrjrFXTt6C zYftZkUrPR0*&*QV{B1q=g_&~pgg!In+%~hG;YKaYWeF`T9IKRB?oVE=df;2i<}Zic z1S@_o6urp~Q?Q_tkty@@vY$E=8D{%OZ}t-9GG}05o$2Y~7_uBG0Dw985-^W0ncd$D z+IED Date: Wed, 27 Mar 2024 11:29:34 +0100 Subject: [PATCH 0008/1048] 10161 - adaptation of Java imports --- .../java/edu/harvard/iq/dataverse/search/IndexServiceBean.java | 1 + .../edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index f032142cf09..e12ee111a18 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -10,6 +10,7 @@ import edu.harvard.iq.dataverse.DatasetFieldConstant; import edu.harvard.iq.dataverse.DatasetFieldServiceBean; import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetFieldValue; import edu.harvard.iq.dataverse.DatasetFieldValueValidator; import edu.harvard.iq.dataverse.DatasetLinkingServiceBean; import edu.harvard.iq.dataverse.DatasetServiceBean; diff --git a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java index bd73f05fea7..cf85899121c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java @@ -1,7 +1,6 @@ package edu.harvard.iq.dataverse.search; import edu.harvard.iq.dataverse.ControlledVocabularyValue; -import edu.harvard.iq.dataverse.DOIServiceBean; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetField; import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; From 2edd0f356b1618bc0f7625cd933264d7495973b1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 23 Apr 2024 09:52:02 -0400 Subject: [PATCH 0009/1048] Fix for #10516 and test --- .../iq/dataverse/pidproviders/PidUtil.java | 8 +++- .../dataverse/pidproviders/PidUtilTest.java | 45 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidUtil.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidUtil.java index 279f18dcd0e..003b4e3f61c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidUtil.java @@ -3,6 +3,7 @@ import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; import edu.harvard.iq.dataverse.pidproviders.handle.HandlePidProvider; +import edu.harvard.iq.dataverse.pidproviders.perma.PermaLinkPidProvider; import edu.harvard.iq.dataverse.util.BundleUtil; import java.io.IOException; import java.io.InputStream; @@ -252,7 +253,12 @@ public static void clearPidProviders() { * Get a PidProvider by protocol/authority/shoulder. */ public static PidProvider getPidProvider(String protocol, String authority, String shoulder) { - return getPidProvider(protocol, authority, shoulder, AbstractPidProvider.SEPARATOR); + switch(protocol) { + case PermaLinkPidProvider.PERMA_PROTOCOL: + return getPidProvider(protocol, authority, shoulder, PermaLinkPidProvider.SEPARATOR); + default: + return getPidProvider(protocol, authority, shoulder, AbstractPidProvider.SEPARATOR); + } } public static PidProvider getPidProvider(String protocol, String authority, String shoulder, String separator) { diff --git a/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java index cffac741c78..cbe48b9780a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java @@ -474,4 +474,49 @@ public void testLegacyConfig() throws IOException { assertEquals(pid1String, pid2.asString()); assertEquals("legacy", pid2.getProviderId()); } + + //Tests support for legacy Perma provider - see #10516 + @Test + @JvmSetting(key = JvmSettings.LEGACY_PERMALINK_BASEURL, value = "http://localhost:8080/") + public void testLegacyPermaConfig() throws IOException { + MockitoAnnotations.openMocks(this); + Mockito.when(settingsServiceBean.getValueForKey(SettingsServiceBean.Key.Shoulder)).thenReturn("FK2"); + Mockito.when(settingsServiceBean.getValueForKey(SettingsServiceBean.Key.Protocol)).thenReturn(PermaLinkPidProvider.PERMA_PROTOCOL); + Mockito.when(settingsServiceBean.getValueForKey(SettingsServiceBean.Key.Authority)).thenReturn("PermaTest"); + + String protocol = settingsServiceBean.getValueForKey(SettingsServiceBean.Key.Protocol); + String authority = settingsServiceBean.getValueForKey(SettingsServiceBean.Key.Authority); + String shoulder = settingsServiceBean.getValueForKey(SettingsServiceBean.Key.Shoulder); + + //Code mirrors the relevant part of PidProviderFactoryBean + if (protocol != null && authority != null && shoulder != null) { + // This line is different than in PidProviderFactoryBean because here we've + // already added the unmanaged providers, so we can't look for null + if (!PidUtil.getPidProvider(protocol, authority, shoulder).canManagePID()) { + PidProvider legacy = null; + // Try to add a legacy provider + String identifierGenerationStyle = settingsServiceBean + .getValueForKey(SettingsServiceBean.Key.IdentifierGenerationStyle, "random"); + String dataFilePidFormat = settingsServiceBean.getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, + "DEPENDENT"); + String baseUrl = JvmSettings.LEGACY_PERMALINK_BASEURL.lookup(); + legacy = new PermaLinkPidProvider("legacy", "legacy", authority, shoulder, + identifierGenerationStyle, dataFilePidFormat, "", "", baseUrl, + PermaLinkPidProvider.SEPARATOR); + if (legacy != null) { + // Not testing parts that require this bean + legacy.setPidProviderServiceBean(null); + PidUtil.addToProviderList(legacy); + } + } else { + System.out.println("Legacy PID provider settings found - ignored since a provider for the same protocol, authority, shoulder has been registered"); + } + + } + //Is a perma PID with the default "" separator recognized? + String pid1String = "perma:PermaTestFK2ABCDEF"; + GlobalId pid2 = PidUtil.parseAsGlobalID(pid1String); + assertEquals(pid1String, pid2.asString()); + assertEquals("legacy", pid2.getProviderId()); + } } From afec2b4984d07d0da96907ca7cf3c2518f448e15 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 23 Apr 2024 09:58:58 -0400 Subject: [PATCH 0010/1048] make base-url optional in legacy config --- .../iq/dataverse/pidproviders/PidProviderFactoryBean.java | 2 +- .../java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java index 40044408c63..746a35600b5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java @@ -203,7 +203,7 @@ private void loadProviders() { passphrase); break; case "perma": - String baseUrl = JvmSettings.LEGACY_PERMALINK_BASEURL.lookup(); + String baseUrl = JvmSettings.LEGACY_PERMALINK_BASEURL.lookupOptional().orElse(SystemConfig.getDataverseSiteUrlStatic()); legacy = new PermaLinkPidProvider("legacy", "legacy", authority, shoulder, identifierGenerationStyle, dataFilePidFormat, "", "", baseUrl, PermaLinkPidProvider.SEPARATOR); diff --git a/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java index cbe48b9780a..3cb7294c0c6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java @@ -499,7 +499,7 @@ public void testLegacyPermaConfig() throws IOException { .getValueForKey(SettingsServiceBean.Key.IdentifierGenerationStyle, "random"); String dataFilePidFormat = settingsServiceBean.getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT"); - String baseUrl = JvmSettings.LEGACY_PERMALINK_BASEURL.lookup(); + String baseUrl = JvmSettings.LEGACY_PERMALINK_BASEURL.lookupOptional().orElse(SystemConfig.getDataverseSiteUrlStatic()); legacy = new PermaLinkPidProvider("legacy", "legacy", authority, shoulder, identifierGenerationStyle, dataFilePidFormat, "", "", baseUrl, PermaLinkPidProvider.SEPARATOR); From 770d0c3851a81b8bb28f3a39a6eba5b4b8872c86 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 23 Apr 2024 09:59:18 -0400 Subject: [PATCH 0011/1048] release note --- doc/release-notes/10516_legacy_permalink_config_fix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/release-notes/10516_legacy_permalink_config_fix.md diff --git a/doc/release-notes/10516_legacy_permalink_config_fix.md b/doc/release-notes/10516_legacy_permalink_config_fix.md new file mode 100644 index 00000000000..d78395252d4 --- /dev/null +++ b/doc/release-notes/10516_legacy_permalink_config_fix.md @@ -0,0 +1 @@ +Support for legacy configuration of a PermaLink PID provider, e.g. using the :Protocol,:Authority, and :Shoulder settings, is fixed. \ No newline at end of file From cb4c586affc435218cfd9f41b069517a97750813 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 23 Apr 2024 17:54:08 -0400 Subject: [PATCH 0012/1048] new changes from QDR --- pom.xml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c4b918318e3..82bb623e7d7 100644 --- a/pom.xml +++ b/pom.xml @@ -108,6 +108,12 @@ io.gdcc sword2-server 2.0.0 + + + xml-apis + xml-apis + + @@ -235,7 +241,7 @@ org.eclipse.parsson jakarta.json - provided + test @@ -557,6 +563,12 @@ org.apache.tika tika-parsers-standard-package ${tika.version} + + + xml-apis + xml-apis + + From 6d0adf5ddba65f46a603bbbe28fcb9ddf22e8b00 Mon Sep 17 00:00:00 2001 From: Patrick Carlson Date: Tue, 3 Jan 2023 10:07:48 -0700 Subject: [PATCH 0013/1048] add check to look for updates to Github actions being used --- .github/dependabot.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..6325029dac1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# Set update schedule for GitHub Actions +# https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/keeping-your-actions-up-to-date-with-dependabot + +version: 2 +updates: + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions daily + interval: "daily" From a0a7c8d05bf5fd0416a48a6052d87c987603312f Mon Sep 17 00:00:00 2001 From: Thibault Coupin Date: Mon, 3 Oct 2022 09:26:24 +0200 Subject: [PATCH 0014/1048] Harvest: map publisher tag to distributorName --- .../db/migration/V5.12.0.1__8739-publisher-during-harvesting.sql | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/main/resources/db/migration/V5.12.0.1__8739-publisher-during-harvesting.sql diff --git a/src/main/resources/db/migration/V5.12.0.1__8739-publisher-during-harvesting.sql b/src/main/resources/db/migration/V5.12.0.1__8739-publisher-during-harvesting.sql new file mode 100644 index 00000000000..c4dbd901181 --- /dev/null +++ b/src/main/resources/db/migration/V5.12.0.1__8739-publisher-during-harvesting.sql @@ -0,0 +1 @@ +update foreignmetadatafieldmapping set datasetfieldname = 'distributorName' where foreignfieldxpath = ':publisher'; From d92d048354d19ca58665f0db6fd0cb673dd6f985 Mon Sep 17 00:00:00 2001 From: plecor <146710476+plecor@users.noreply.github.com> Date: Thu, 2 May 2024 13:40:00 +0200 Subject: [PATCH 0015/1048] Rename migration file --- ...vesting.sql => V6.2.0.2__8739-publisher-during-harvesting.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.12.0.1__8739-publisher-during-harvesting.sql => V6.2.0.2__8739-publisher-during-harvesting.sql} (100%) diff --git a/src/main/resources/db/migration/V5.12.0.1__8739-publisher-during-harvesting.sql b/src/main/resources/db/migration/V6.2.0.2__8739-publisher-during-harvesting.sql similarity index 100% rename from src/main/resources/db/migration/V5.12.0.1__8739-publisher-during-harvesting.sql rename to src/main/resources/db/migration/V6.2.0.2__8739-publisher-during-harvesting.sql From d6a6e56df7bae324e95c25e7a2b0de5f6d273aa1 Mon Sep 17 00:00:00 2001 From: plecor <146710476+plecor@users.noreply.github.com> Date: Tue, 14 May 2024 15:31:20 +0200 Subject: [PATCH 0016/1048] Add use case to HarvestingClientsIT --- .../edu/harvard/iq/dataverse/api/HarvestingClientsIT.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java index 340eab161bb..5020e37edb8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java @@ -299,6 +299,11 @@ private void harvestingClientRun(boolean allowHarvestingMissingCVV) throws Inte } // verify count after collecting global ids assertEquals(expectedNumberOfSetsHarvested, jsonPath.getInt("data.total_count")); + + // ensure the publisher name is present in the harvested dataset citation + Response harvestedDataverse = given().get(ARCHIVE_URL + "/api/dataverses/1"); + String harvestedDataverseName = harvestedDataverse.getBody().jsonPath().getString("data.name"); + assertTrue(jsonPath.getString("data.items[0].citation").contains(harvestedDataverseName)); // Fail if it hasn't completed in maxWait seconds assertTrue(i < maxWait); From 64b69b94a7677d927c81b30f1a9cf43b470412eb Mon Sep 17 00:00:00 2001 From: plecor <146710476+plecor@users.noreply.github.com> Date: Tue, 14 May 2024 15:50:19 +0200 Subject: [PATCH 0017/1048] Add release note --- doc/release-notes/8739-publisher-during-harvesting.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/release-notes/8739-publisher-during-harvesting.md diff --git a/doc/release-notes/8739-publisher-during-harvesting.md b/doc/release-notes/8739-publisher-during-harvesting.md new file mode 100644 index 00000000000..602b2cf34d6 --- /dev/null +++ b/doc/release-notes/8739-publisher-during-harvesting.md @@ -0,0 +1 @@ +The publisher value of harvested datasets is now attributed to the dataset's distributor instead of its producer. This change affects all newly harvested datasets. For more information, see #8739 \ No newline at end of file From 5dfb01ef58f11dbac8ed4153e265e606b55fef0c Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 14 May 2024 13:55:24 -0400 Subject: [PATCH 0018/1048] add change in behavior to new harvesting client changlog #8739 --- doc/sphinx-guides/source/admin/harvestclients.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index 59fc4dc2c64..1e45fd06fc8 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -47,3 +47,8 @@ What if a Run Fails? Each harvesting client run logs a separate file per run to the app server's default logging directory (``/usr/local/payara6/glassfish/domains/domain1/logs/`` unless you've changed it). Look for filenames in the format ``harvest_TARGET_YYYY_MM_DD_timestamp.log`` to get a better idea of what's going wrong. Note that you'll want to run a minimum of Dataverse Software 4.6, optimally 4.18 or beyond, for the best OAI-PMH interoperability. + +Harvesting Client Changelog +--------------------------- + +- As of Dataverse 6.3, the publisher value of harvested datasets is now attributed to the dataset's distributor instead of its producer. This change affects all newly harvested datasets. For more information, see https://github.com/IQSS/dataverse/pull/9013 From 007b2d4bcd726853fcb6cc7fa028e806b7565d80 Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Thu, 23 May 2024 15:24:14 +0200 Subject: [PATCH 0019/1048] Added endpoint to update global role + fixed issue with GUI custom role edition --- .../dataverse/DataverseRoleServiceBean.java | 31 ++++++++++--------- .../edu/harvard/iq/dataverse/api/Admin.java | 16 ++++++++++ .../harvard/iq/dataverse/api/dto/RoleDTO.java | 8 +++-- .../command/impl/CreateRoleCommand.java | 12 +++---- 4 files changed, 44 insertions(+), 23 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java index 78d5eaf3414..7ee6b7295a8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java @@ -23,7 +23,6 @@ import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; import jakarta.persistence.TypedQuery; -//import jakarta.validation.constraints.NotNull; /** * @@ -40,6 +39,9 @@ public class DataverseRoleServiceBean implements java.io.Serializable { @EJB RoleAssigneeServiceBean roleAssigneeService; + + @EJB + DataverseServiceBean dataverseService; @EJB IndexServiceBean indexService; @EJB @@ -48,22 +50,21 @@ public class DataverseRoleServiceBean implements java.io.Serializable { IndexAsync indexAsync; public DataverseRole save(DataverseRole aRole) { - if (aRole.getId() == null) { + if (aRole.getId() == null) { // persist a new Role em.persist(aRole); - /** - * @todo Why would getId be null? Should we call - * indexDefinitionPoint here too? A: it's null for new roles. - */ - return aRole; - } else { - DataverseRole merged = em.merge(aRole); - /** - * @todo update permissionModificationTime here. - */ - IndexResponse indexDefinitionPountResult = indexDefinitionPoint(merged.getOwner()); - logger.info("aRole getId was not null. Indexing result: " + indexDefinitionPountResult); - return merged; + } else { // update an existing Role + aRole = em.merge(aRole); } + + DvObject owner = aRole.getOwner(); + if(owner == null) { // Builtin Role + owner = dataverseService.findByAlias("root"); + } + + IndexResponse indexDefinitionPointResult = indexDefinitionPoint(owner); + logger.info("Indexing result: " + indexDefinitionPointResult); + + return aRole; } public RoleAssignment save(RoleAssignment assignment) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 802904b5173..d95c5ca967a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1007,6 +1007,22 @@ public Response createNewBuiltinRole(RoleDTO roleDto) { actionLogSvc.log(alr); } } + @Path("roles/{id}") + @PUT + public Response updateBuiltinRole(RoleDTO roleDto, @PathParam("id") long roleId) { + ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "updateBuiltInRole") + .setInfo(roleDto.getAlias() + ":" + roleDto.getDescription()); + try { + DataverseRole role = roleDto.updateRoleFromDTO(rolesSvc.find(roleId)); + return ok(json(rolesSvc.save(role))); + } catch (Exception e) { + alr.setActionResult(ActionLogRecord.Result.InternalError); + alr.setInfo(alr.getInfo() + "// " + e.getMessage()); + return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } finally { + actionLogSvc.log(alr); + } + } @Path("roles") @GET diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/RoleDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/RoleDTO.java index 58e30ade584..5769ab430ad 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/dto/RoleDTO.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/RoleDTO.java @@ -47,11 +47,11 @@ public void setPermissions(String[] permissions) { this.permissions = permissions; } - public DataverseRole asRole() { - DataverseRole r = new DataverseRole(); + public DataverseRole updateRoleFromDTO(DataverseRole r) { r.setAlias(alias); r.setDescription(description); r.setName(name); + r.clearPermissions(); if (permissions != null) { if (permissions.length > 0) { if (permissions[0].trim().toLowerCase().equals("all")) { @@ -65,5 +65,9 @@ public DataverseRole asRole() { } return r; } + + public DataverseRole asRole() { + return updateRoleFromDTO(new DataverseRole()); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommand.java index 8cffcd3d821..08923125bdf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommand.java @@ -22,12 +22,12 @@ @RequiredPermissions(Permission.ManageDataversePermissions) public class CreateRoleCommand extends AbstractCommand { - private final DataverseRole created; + private final DataverseRole role; private final Dataverse dv; public CreateRoleCommand(DataverseRole aRole, DataverseRequest aRequest, Dataverse anAffectedDataverse) { super(aRequest, anAffectedDataverse); - created = aRole; + role = aRole; dv = anAffectedDataverse; } @@ -41,16 +41,16 @@ public DataverseRole execute(CommandContext ctxt) throws CommandException { //Test to see if the role already exists in DB try { DataverseRole testRole = ctxt.em().createNamedQuery("DataverseRole.findDataverseRoleByAlias", DataverseRole.class) - .setParameter("alias", created.getAlias()) + .setParameter("alias", role.getAlias()) .getSingleResult(); - if (!(testRole == null)) { + if (testRole != null && !testRole.getId().equals(role.getId())) { throw new IllegalCommandException(BundleUtil.getStringFromBundle("permission.role.not.created.alias.already.exists"), this); } } catch (NoResultException nre) { // we want no results because that meand we can create a role } - dv.addRole(created); - return ctxt.roles().save(created); + dv.addRole(role); + return ctxt.roles().save(role); } } From c29a9af03f955807b2b6bee411eb1ac3303db52e Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Wed, 5 Jun 2024 17:46:20 +0200 Subject: [PATCH 0020/1048] Update Native API guide --- doc/sphinx-guides/source/api/native-api.rst | 26 +++++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 8c54a937353..497397a02d7 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -4249,12 +4249,12 @@ The JSON representation of a role (``roles.json``) looks like this:: { "alias": "sys1", - "name": “Restricted System Role”, - "description": “A person who may only add datasets.”, + "name": "Restricted System Role", + "description": "A person who may only add datasets.", "permissions": [ "AddDataset" ] - } + } .. note:: alias is constrained to a length of 16 characters @@ -5313,9 +5313,25 @@ Creates a global role in the Dataverse installation. The data POSTed are assumed export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org - export ID=root - curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/admin/roles" --upload-file roles.json + curl -H "Content-Type: application/json" -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/admin/roles" --upload-file roles.json + +``roles.json`` see :ref:`json-representation-of-a-role` + +Update Global Role +~~~~~~~~~~~~~~~~~~ + +Update a global role in the Dataverse installation. The data PUTed are assumed to be a role JSON. :: + + POST http://$SERVER/api/admin/roles + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=24 + + curl -H "Content-Type: application/json" -H "X-Dataverse-key:$API_TOKEN" -X PUT "$SERVER_URL/api/admin/roles/$ID" --upload-file roles.json ``roles.json`` see :ref:`json-representation-of-a-role` From 7d55ae17b4f6a4ae74d7118d17817e0f3e27b9a4 Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Wed, 5 Jun 2024 18:23:08 +0200 Subject: [PATCH 0021/1048] Adding release note --- doc/release-notes/8808-10575-update-global-role.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/release-notes/8808-10575-update-global-role.md diff --git a/doc/release-notes/8808-10575-update-global-role.md b/doc/release-notes/8808-10575-update-global-role.md new file mode 100644 index 00000000000..4b50a6079dc --- /dev/null +++ b/doc/release-notes/8808-10575-update-global-role.md @@ -0,0 +1,11 @@ +## Release Highlights + +### Update a Global Role + +A new API endpoint has been added that allow the update a global role. See [Native API Guide > Update Global Role](https://guides.dataverse.org/en/6.3/api/native-api.html#update-global-role) (#10612) + +## Bug fixes + +### Edition of custom role fixed + +It is now possible to edit a custom role with the same alias (reported in #8808) \ No newline at end of file From 9d0004dbea2625ecd7382af6ad30eac898c33a5d Mon Sep 17 00:00:00 2001 From: luddaniel <83018819+luddaniel@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:31:47 +0200 Subject: [PATCH 0022/1048] Update doc/release-notes/8808-10575-update-global-role.md Co-authored-by: Oliver Bertuch --- doc/release-notes/8808-10575-update-global-role.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/8808-10575-update-global-role.md b/doc/release-notes/8808-10575-update-global-role.md index 4b50a6079dc..38142f972e8 100644 --- a/doc/release-notes/8808-10575-update-global-role.md +++ b/doc/release-notes/8808-10575-update-global-role.md @@ -2,7 +2,7 @@ ### Update a Global Role -A new API endpoint has been added that allow the update a global role. See [Native API Guide > Update Global Role](https://guides.dataverse.org/en/6.3/api/native-api.html#update-global-role) (#10612) +A new API endpoint has been added that allows a global role to be updated. See [Native API Guide > Update Global Role](https://guides.dataverse.org/en/6.3/api/native-api.html#update-global-role) (#10612) ## Bug fixes From cc4f52b680c90703a209ac261f4c99f278c1a2c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asbj=C3=B8rn=20Sk=C3=B8dt?= Date: Wed, 12 Jun 2024 10:02:41 +0200 Subject: [PATCH 0023/1048] added archiva.tsv metadata block --- scripts/api/data/metadatablocks/archival.tsv | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 scripts/api/data/metadatablocks/archival.tsv diff --git a/scripts/api/data/metadatablocks/archival.tsv b/scripts/api/data/metadatablocks/archival.tsv new file mode 100644 index 00000000000..6380514bcc0 --- /dev/null +++ b/scripts/api/data/metadatablocks/archival.tsv @@ -0,0 +1,19 @@ +#metadataBlock name dataverseAlias displayName blockURI + archive Archival Metadata +#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id termURI + archiveSubmitToArchivalAppraisal Archival Appraisal An assessment whether the dataset should be submitted for archival appraisal text 0 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE archive + archiveArchivedFrom Archived from A date (YYYY-MM-DD) from which the dataset is archived YYYY-MM-DD date 2 #VALUE TRUE FALSE FALSE FALSE FALSE FALSE archive + archiveArchivedFor Retention period The retention period for which the archived dataset is to be kept by the holding archive text 3 #VALUE TRUE TRUE FALSE FALSE FALSE FALSE archive + archiveHoldingArchive Archived at Holding Archive The holding archive where the dataset is archived text 4 #VALUE FALSE FALSE TRUE FALSE FALSE FALSE archive https://schema.org/holdingArchive + archiveArchivedAt Archived at URL URL to holding archive where the dataset is archived URL url 5 #VALUE FALSE FALSE TRUE FALSE FALSE FALSE archive https://schema.org/archivedAt +#controlledVocabulary DatasetField Value identifier displayOrder + archiveArchivedFor 1 year 1_year 0 + archiveArchivedFor 3 years 3_years 1 + archiveArchivedFor 5 years 5_years 2 + archiveArchivedFor 10 years 10_years 3 + archiveArchivedFor 20 years 20_years 4 + archiveArchivedFor Indefinitely indefinitely 5 + archiveArchivedFor Unknown unknown 6 + archiveSubmitToArchivalAppraisal Yes yes 0 + archiveSubmitToArchivalAppraisal No no 1 + archiveSubmitToArchivalAppraisal Unknown unknown 2 From 52d72d38a717cd436b07ac4bab37f2e427a42717 Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Wed, 12 Jun 2024 10:07:21 +0200 Subject: [PATCH 0024/1048] Update doc after review --- doc/sphinx-guides/source/api/native-api.rst | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 497397a02d7..ef02c572efb 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5312,7 +5312,7 @@ Creates a global role in the Dataverse installation. The data POSTed are assumed .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - export SERVER_URL=https://demo.dataverse.org + export SERVER_URL=http://localhost:8080 curl -H "Content-Type: application/json" -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/admin/roles" --upload-file roles.json @@ -5321,29 +5321,34 @@ Creates a global role in the Dataverse installation. The data POSTed are assumed Update Global Role ~~~~~~~~~~~~~~~~~~ -Update a global role in the Dataverse installation. The data PUTed are assumed to be a role JSON. :: +Update a global role in the Dataverse installation. The PUTed data is assumed to be a complete JSON role as it will overwrite the existing role. :: - POST http://$SERVER/api/admin/roles + PUT http://$SERVER/api/admin/roles/$ID + +A curl example using an ``ID`` .. code-block:: bash - export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - export SERVER_URL=https://demo.dataverse.org + export SERVER_URL=http://localhost:8080 export ID=24 - curl -H "Content-Type: application/json" -H "X-Dataverse-key:$API_TOKEN" -X PUT "$SERVER_URL/api/admin/roles/$ID" --upload-file roles.json + curl -H "Content-Type: application/json" -X PUT "$SERVER_URL/api/admin/roles/$ID" --upload-file roles.json ``roles.json`` see :ref:`json-representation-of-a-role` Delete Global Role ~~~~~~~~~~~~~~~~~~ +Deletes an ``DataverseRole`` whose ``id`` is passed. :: + + DELETE http://$SERVER/api/admin/roles/$ID + A curl example using an ``ID`` .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - export SERVER_URL=https://demo.dataverse.org + export SERVER_URL=http://localhost:8080 export ID=24 curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/admin/roles/$ID" From ef5952f8318052c4aa41067b4960627699e34cee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asbj=C3=B8rn=20Sk=C3=B8dt?= Date: Tue, 18 Jun 2024 13:49:11 +0000 Subject: [PATCH 0025/1048] changed metadata title from "archive" to "archival" and removed "archive" as prefix to all fields --- scripts/api/data/metadatablocks/archival.tsv | 32 ++++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/api/data/metadatablocks/archival.tsv b/scripts/api/data/metadatablocks/archival.tsv index 6380514bcc0..07fa14fdc9e 100644 --- a/scripts/api/data/metadatablocks/archival.tsv +++ b/scripts/api/data/metadatablocks/archival.tsv @@ -1,19 +1,19 @@ #metadataBlock name dataverseAlias displayName blockURI - archive Archival Metadata + archival Archival Metadata #datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id termURI - archiveSubmitToArchivalAppraisal Archival Appraisal An assessment whether the dataset should be submitted for archival appraisal text 0 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE archive - archiveArchivedFrom Archived from A date (YYYY-MM-DD) from which the dataset is archived YYYY-MM-DD date 2 #VALUE TRUE FALSE FALSE FALSE FALSE FALSE archive - archiveArchivedFor Retention period The retention period for which the archived dataset is to be kept by the holding archive text 3 #VALUE TRUE TRUE FALSE FALSE FALSE FALSE archive - archiveHoldingArchive Archived at Holding Archive The holding archive where the dataset is archived text 4 #VALUE FALSE FALSE TRUE FALSE FALSE FALSE archive https://schema.org/holdingArchive - archiveArchivedAt Archived at URL URL to holding archive where the dataset is archived URL url 5 #VALUE FALSE FALSE TRUE FALSE FALSE FALSE archive https://schema.org/archivedAt + submitToArchivalAppraisal Archival Appraisal An assessment whether the dataset should be submitted for archival appraisal text 0 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE archive + archivedFrom Archived from A date (YYYY-MM-DD) from which the dataset is archived YYYY-MM-DD date 2 #VALUE TRUE FALSE FALSE FALSE FALSE FALSE archive + archivedFor Retention period The retention period for which the archived dataset is to be kept by the holding archive text 3 #VALUE TRUE TRUE FALSE FALSE FALSE FALSE archive + holdingArchive Archived at Holding Archive The holding archive where the dataset is archived text 4 #VALUE FALSE FALSE TRUE FALSE FALSE FALSE archive https://schema.org/holdingArchive + archivedAt Archived at URL URL to holding archive where the dataset is archived URL url 5 #VALUE FALSE FALSE TRUE FALSE FALSE FALSE archive https://schema.org/archivedAt #controlledVocabulary DatasetField Value identifier displayOrder - archiveArchivedFor 1 year 1_year 0 - archiveArchivedFor 3 years 3_years 1 - archiveArchivedFor 5 years 5_years 2 - archiveArchivedFor 10 years 10_years 3 - archiveArchivedFor 20 years 20_years 4 - archiveArchivedFor Indefinitely indefinitely 5 - archiveArchivedFor Unknown unknown 6 - archiveSubmitToArchivalAppraisal Yes yes 0 - archiveSubmitToArchivalAppraisal No no 1 - archiveSubmitToArchivalAppraisal Unknown unknown 2 + archivedFor 1 year 1_year 0 + archivedFor 3 years 3_years 1 + archivedFor 5 years 5_years 2 + archivedFor 10 years 10_years 3 + archivedFor 20 years 20_years 4 + archivedFor Indefinitely indefinitely 5 + archivedFor Unknown unknown 6 + submitToArchivalAppraisal Yes yes 0 + submitToArchivalAppraisal No no 1 + submitToArchivalAppraisal Unknown unknown 2 From 1eceaeef0ac2e2f435e8238c9804d67d4d636f4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Wed, 26 Jun 2024 18:15:02 +0200 Subject: [PATCH 0026/1048] fix compilation failure after merge --- .../java/edu/harvard/iq/dataverse/search/IndexServiceBean.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 0c594653870..6e593eb223b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -25,6 +25,7 @@ import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.PermissionServiceBean; +import edu.harvard.iq.dataverse.Retention; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; import edu.harvard.iq.dataverse.batch.util.LoggingUtil; From 0fd9fce1c9dfbbac9e69961c70fceba8eeef6a48 Mon Sep 17 00:00:00 2001 From: Dimitri Szabo <46443753+DS-INRA@users.noreply.github.com> Date: Wed, 3 Jul 2024 15:07:31 +0200 Subject: [PATCH 0027/1048] Update README.md Added sections, emojis and more links. Epanded the text --- README.md | 51 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 77720453d5f..60359e7c618 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,55 @@ Dataverse® =============== -Dataverse is an [open source][] software platform for sharing, finding, citing, and preserving research data (developed by the [Dataverse team](https://dataverse.org/about) at the [Institute for Quantitative Social Science](https://iq.harvard.edu/) and the [Dataverse community][]). +Welcome to Dataverse®, the [open source][] software platform designed for sharing, finding, citing, and preserving research data. Developed by the Dataverse team at the [Institute for Quantitative Social Science](https://iq.harvard.edu/) and the [Dataverse community][], our platform is built to support researchers and institutions in sharing, finding, citing, and preserving research their data effectively. -[dataverse.org][] is our home on the web and shows a map of Dataverse installations around the world, a list of [features][], [integrations][] that have been made possible through [REST APIs][], our [project board][], our development [roadmap][], and more. +## ✔ Try Dataverse -We maintain a demo site at [demo.dataverse.org][] which you are welcome to use for testing and evaluating Dataverse. +We invite you to explore our demo site at [demo.dataverse.org][]. This site is ideal for testing and evaluating Dataverse in a risk-free environment. -To install Dataverse, please see our [Installation Guide][] which will prompt you to download our [latest release][]. Docker users should consult the [Container Guide][]. +## 🌐 Our Web Presence -To discuss Dataverse with the community, please join our [mailing list][], participate in a [community call][], chat with us at [chat.dataverse.org][], or attend our annual [Dataverse Community Meeting][]. +Visit [dataverse.org][], our home on the web, for a comprehensive overview of Dataverse. Here, you will find: -We love contributors! Please see our [Contributing Guide][] for ways you can help. +- An interactive map showcasing Dataverse installations worldwide. +- A detailed list of [features][]. +- Information on [integrations][] that have been made possible through our [REST APIs][]. +- Our [project board][] and development [roadmap][]. +- News, events, and more. + +## 📥 Installation + +Ready to get started? Follow our [Installation Guide][] to download and install the latest release of Dataverse. + +If you are using Docker, please refer to our [Container Guide][] for detailed instructions. + +## 🏘 Community and Support + +Engage with the vibrant Dataverse community through various channels: + +- **[Mailing List][]**: Join the conversation on our [mailing list][]. +- **[Community Calls][]**: Participate in our regular [community calls][] to discuss new features, ask questions, and share your experiences. +- **[Chat][]**: Connect with us and other users in real-time at [chat.dataverse.org][]. +- **[Dataverse Community Meeting][]**: Attend our annual [Dataverse Community Meeting][] to network, learn, and collaborate with peers and experts. +- **[DataverseTV][]**: Watch the video content from the Dataverse community on [DataverseTV][] and on [Harvard's IQSS YouTube channel][]. + +## 🧑‍💻️ Contribute to Dataverse + +We love contributors! Whether you are a developer, researcher, or enthusiast, there are many ways you can help. + +Visit our [Contributing Guide][] to learn how you can get involved. + +Join us in building and enhancing Dataverse to make research data more accessible and impactful. Your support and participation are crucial to our success! + +## ⚖️ Legal Information Dataverse is a trademark of President and Fellows of Harvard College and is registered in the United States. +--- +For more detailed information, visit our website at [dataverse.org][]. + +Feel free to reach out with any questions or feedback. Happy researching! + [![Dataverse Project logo](src/main/webapp/resources/images/dataverseproject_logo.jpg "Dataverse Project")](http://dataverse.org) [![API Test Status](https://jenkins.dataverse.org/buildStatus/icon?job=IQSS-dataverse-develop&subject=API%20Test%20Status)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/) @@ -37,6 +72,10 @@ Dataverse is a trademark of President and Fellows of Harvard College and is regi [Contributing Guide]: CONTRIBUTING.md [mailing list]: https://groups.google.com/group/dataverse-community [community call]: https://dataverse.org/community-calls +[Chat]: https://chat.dataverse.org [chat.dataverse.org]: https://chat.dataverse.org [Dataverse Community Meeting]: https://dataverse.org/events [open source]: LICENSE.md +[community calls]: https://dataverse.org/community-calls +[DataverseTV]: https://dataverse.org/dataversetv +[Harvard's IQSS YouTube channel]: https://www.youtube.com/@iqssatharvarduniversity8672 From e3f47c5b9ce5dd79a6dc439289dfb7148ea45d47 Mon Sep 17 00:00:00 2001 From: Dimitri Szabo <46443753+DS-INRA@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:27:50 +0200 Subject: [PATCH 0028/1048] Add logo and Table of contents --- README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/README.md b/README.md index 60359e7c618..ef184e835f2 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,38 @@ Dataverse® =============== +
+

+ +## Table of Contents + +
    +
  1. ❓ What is Dataverse?
    2. +
  2. ✔ Try Dataverse
    3. +
  3. 🌐 Our Web Presence
    4. +
  4. 📥 Installation
    5. +
  5. 🏘 Community and Support
    6. +
  6. 🧑‍💻️ Contributing
    7. +
  7. ⚖️ Legal Information
    8. +
+ + + +## ❓ What is Dataverse? Welcome to Dataverse®, the [open source][] software platform designed for sharing, finding, citing, and preserving research data. Developed by the Dataverse team at the [Institute for Quantitative Social Science](https://iq.harvard.edu/) and the [Dataverse community][], our platform is built to support researchers and institutions in sharing, finding, citing, and preserving research their data effectively. + + ## ✔ Try Dataverse We invite you to explore our demo site at [demo.dataverse.org][]. This site is ideal for testing and evaluating Dataverse in a risk-free environment. + + ## 🌐 Our Web Presence Visit [dataverse.org][], our home on the web, for a comprehensive overview of Dataverse. Here, you will find: @@ -17,12 +43,16 @@ Visit [dataverse.org][], our home on the web, for a comprehensive overview of Da - Our [project board][] and development [roadmap][]. - News, events, and more. + + ## 📥 Installation Ready to get started? Follow our [Installation Guide][] to download and install the latest release of Dataverse. If you are using Docker, please refer to our [Container Guide][] for detailed instructions. + + ## 🏘 Community and Support Engage with the vibrant Dataverse community through various channels: @@ -33,6 +63,7 @@ Engage with the vibrant Dataverse community through various channels: - **[Dataverse Community Meeting][]**: Attend our annual [Dataverse Community Meeting][] to network, learn, and collaborate with peers and experts. - **[DataverseTV][]**: Watch the video content from the Dataverse community on [DataverseTV][] and on [Harvard's IQSS YouTube channel][]. + ## 🧑‍💻️ Contribute to Dataverse We love contributors! Whether you are a developer, researcher, or enthusiast, there are many ways you can help. @@ -41,6 +72,7 @@ Visit our [Contributing Guide][] to learn how you can get involved. Join us in building and enhancing Dataverse to make research data more accessible and impactful. Your support and participation are crucial to our success! + ## ⚖️ Legal Information Dataverse is a trademark of President and Fellows of Harvard College and is registered in the United States. From e8a9db28b45c1e55ed7fcd4afc07c7778597d253 Mon Sep 17 00:00:00 2001 From: Dimitri Szabo <46443753+DS-INRA@users.noreply.github.com> Date: Fri, 19 Jul 2024 23:48:49 +0200 Subject: [PATCH 0029/1048] Updated ToC --- README.md | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index ef184e835f2..54ad2f780ab 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,17 @@ Dataverse® =============== -
- + +![309827846-6c4d79e4-7be5-4102-88bd-dfa167dc79d3](https://github.com/user-attachments/assets/cd5d0e97-47cf-45be-9639-179839adafe0) ## Table of Contents -
    -
  1. ❓ What is Dataverse?
    2. -
  2. ✔ Try Dataverse
    3. -
  3. 🌐 Our Web Presence
    4. -
  4. 📥 Installation
    5. -
  5. 🏘 Community and Support
    6. -
  6. 🧑‍💻️ Contributing
    7. -
  7. ⚖️ Legal Information
    8. -
+1. [❓ What is Dataverse?](#what-is-dataverse) +2. [✔ Try Dataverse](#try-dataverse) +3. [🌐 Our Web Presence](#our-web-presence) +4. [📥 Installation](#installation) +5. [🏘 Community and Support](#community-and-support) +6. [🧑‍💻️ Contributing](#contributing) +7. [⚖️ Legal Information](#legal-informations) From a013340b36ab8db916cbe8dc5eb81c23d88f01e4 Mon Sep 17 00:00:00 2001 From: Dimitri Szabo <46443753+DS-INRA@users.noreply.github.com> Date: Fri, 19 Jul 2024 23:54:33 +0200 Subject: [PATCH 0030/1048] Fix Logo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 54ad2f780ab..850a6c5eed1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Dataverse® =============== -![309827846-6c4d79e4-7be5-4102-88bd-dfa167dc79d3](https://github.com/user-attachments/assets/cd5d0e97-47cf-45be-9639-179839adafe0) +![Dataverse-logo](https://github.com/IQSS/dataverse-frontend/assets/7512607/6c4d79e4-7be5-4102-88bd-dfa167dc79d3) ## Table of Contents From 6cca160c9d91679bb85ae597f40e66746723b906 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Sun, 21 Jul 2024 13:16:04 -0400 Subject: [PATCH 0031/1048] solr 9.6 and other lib updates --- conf/solr/solrconfig.xml | 2 +- modules/dataverse-parent/pom.xml | 2 +- pom.xml | 26 ++++++++++++-------------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/conf/solr/solrconfig.xml b/conf/solr/solrconfig.xml index 34386375fe1..2115cb1f822 100644 --- a/conf/solr/solrconfig.xml +++ b/conf/solr/solrconfig.xml @@ -35,7 +35,7 @@ that you fully re-index after changing this setting as it can affect both how text is indexed and queried. --> - 9.7 + 9.10 6.2024.6 42.7.2 - 9.4.1 + 9.6.1 1.12.748 26.30.0 diff --git a/pom.xml b/pom.xml index 76a8f61444f..9f0eaaa11ce 100644 --- a/pom.xml +++ b/pom.xml @@ -29,8 +29,8 @@ 1.2.18.4 9.22.1 1.20.1 - 5.2.1 - 2.4.1 + 5.2.5 + 2.9.2 5.5.3 Dataverse API @@ -136,12 +136,12 @@ com.apicatalog titanium-json-ld - 1.3.2 + 1.4.0 com.google.code.gson gson - 2.8.9 + 2.9.1 compile @@ -157,11 +157,9 @@ provided - - org.everit.json - org.everit.json.schema - 1.5.1 + com.github.erosb + everit-json-schema + 1.14.1 org.mindrot @@ -319,7 +317,7 @@ org.apache.solr solr-solrj - 9.4.1 + 9.6.1 colt @@ -390,7 +388,7 @@ com.github.jai-imageio jai-imageio-core - 1.3.1 + 1.4.0 org.ocpsoft.rewrite @@ -474,7 +472,7 @@ com.google.auto.service auto-service - 1.0-rc2 + 1.1.1 true jar @@ -553,7 +551,7 @@ org.xmlunit xmlunit-core - 2.9.1 + 2.10.0 com.google.cloud @@ -615,7 +613,7 @@ org.xmlunit xmlunit-assertj3 - 2.8.2 + 2.10.0 test From d19737391a5b3c014b291028fb2d93f44ed2fcba Mon Sep 17 00:00:00 2001 From: Florian Fritze Date: Fri, 9 Aug 2024 10:27:57 +0200 Subject: [PATCH 0032/1048] bugfix: create correct json output for metadatablock api call --- .../iq/dataverse/util/json/JsonPrinter.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index c72dfc1d127..4107b8e3d45 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -55,7 +55,7 @@ import jakarta.ejb.Singleton; import jakarta.json.JsonArray; import jakarta.json.JsonObject; -import java.math.BigDecimal; +import java.util.function.Predicate; /** * Convert objects to Json. @@ -639,9 +639,13 @@ public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printO jsonObjectBuilder.add("displayOnCreate", metadataBlock.isDisplayOnCreate()); JsonObjectBuilder fieldsBuilder = Json.createObjectBuilder(); - Set datasetFieldTypes = new TreeSet<>(metadataBlock.getDatasetFieldTypes()); - - for (DatasetFieldType datasetFieldType : datasetFieldTypes) { + + Predicate isNoChild = element -> element.isChild() == false; + List childLessList = metadataBlock.getDatasetFieldTypes().stream().filter(isNoChild).toList(); + Set datasetFieldTypesNoChildSorted = new TreeSet<>(childLessList); + + for (DatasetFieldType datasetFieldType : datasetFieldTypesNoChildSorted) { + Long datasetFieldTypeId = datasetFieldType.getId(); boolean requiredAsInputLevelInOwnerDataverse = ownerDataverse != null && ownerDataverse.isDatasetFieldTypeRequiredAsInputLevel(datasetFieldTypeId); boolean includedAsInputLevelInOwnerDataverse = ownerDataverse != null && ownerDataverse.isDatasetFieldTypeIncludedAsInputLevel(datasetFieldTypeId); @@ -658,7 +662,7 @@ public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printO fieldsBuilder.add(datasetFieldType.getName(), json(datasetFieldType, ownerDataverse)); } } - + jsonObjectBuilder.add("fields", fieldsBuilder); return jsonObjectBuilder; } From db1c59c26dfd241c175e545543ffab60f351284e Mon Sep 17 00:00:00 2001 From: Florian Fritze Date: Wed, 14 Aug 2024 08:11:48 +0200 Subject: [PATCH 0033/1048] added docu for the fix --- doc/release-notes/master_json_fix.md | 1 + doc/sphinx-guides/source/api/changelog.rst | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 doc/release-notes/master_json_fix.md diff --git a/doc/release-notes/master_json_fix.md b/doc/release-notes/master_json_fix.md new file mode 100644 index 00000000000..aa30b90c2cb --- /dev/null +++ b/doc/release-notes/master_json_fix.md @@ -0,0 +1 @@ +This pull request fixes an issue in the JsonPrinter class so that there are no duplicated entries in the JSON metadata or ommitted metadata properties. After the fix is applied the /api/metadatablocks/ endpoint should return correct JSON. \ No newline at end of file diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index a7af3e84b28..7f210af0df7 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -7,6 +7,11 @@ This API changelog is experimental and we would love feedback on its usefulness. :local: :depth: 1 +v6.4 +---- + +- /api/metadatablocks is now returning no duplicated metadata properties and does not ommit metadata properties when called. The JsonPrinter class out is fixed. + v6.3 ---- From 9c480a35b7081776fdcb8797b9ed37a46c184c16 Mon Sep 17 00:00:00 2001 From: Florian Fritze Date: Wed, 14 Aug 2024 08:14:07 +0200 Subject: [PATCH 0034/1048] corrected a word --- doc/sphinx-guides/source/api/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 7f210af0df7..378cdb9f047 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -10,7 +10,7 @@ This API changelog is experimental and we would love feedback on its usefulness. v6.4 ---- -- /api/metadatablocks is now returning no duplicated metadata properties and does not ommit metadata properties when called. The JsonPrinter class out is fixed. +- /api/metadatablocks is now returning no duplicated metadata properties and does not ommit metadata properties when called. The JsonPrinter class output is fixed. v6.3 ---- From 955312df6710841cbb4989326a28a067a07f6b5a Mon Sep 17 00:00:00 2001 From: darms Date: Mon, 19 Aug 2024 13:30:43 +0200 Subject: [PATCH 0035/1048] feat(api/versions): Added a new optional property to hide metadataBlocks from API response. --- .../10171-exlude-metadatablocks.md | 4 ++++ doc/sphinx-guides/source/api/native-api.rst | 8 +++++++ .../harvard/iq/dataverse/api/Datasets.java | 11 +++++---- .../iq/dataverse/util/json/JsonPrinter.java | 23 ++++++++++++------- 4 files changed, 34 insertions(+), 12 deletions(-) create mode 100644 doc/release-notes/10171-exlude-metadatablocks.md diff --git a/doc/release-notes/10171-exlude-metadatablocks.md b/doc/release-notes/10171-exlude-metadatablocks.md new file mode 100644 index 00000000000..64fadfcc35e --- /dev/null +++ b/doc/release-notes/10171-exlude-metadatablocks.md @@ -0,0 +1,4 @@ +Extension of API `{id}/versions` and `{id}/versions/{versionId}` with an optional ``excludeMetadataBlocks`` parameter, +that specifies whether the metadataBlocks should be listed in the output. It defaults to ``true``, preserving backward +compatibility. (Note that for a dataset with a large number of versions and/or metadataBlocks having the metadata blocks +included can dramatically increase the volume of the output). diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index a5f7d03899a..5168b8c5917 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1150,6 +1150,8 @@ It returns a list of versions with their metadata, and file list: The optional ``excludeFiles`` parameter specifies whether the files should be listed in the output. It defaults to ``true``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or files having the files included can dramatically increase the volume of the output). A separate ``/files`` API can be used for listing the files, or a subset thereof in a given version. +The optional ``excludeMetadataBlocks`` parameter specifies whether the metadataBlocks should be listed in the output. It defaults to ``true``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or metadataBlocks having the metadata blocks included can dramatically increase the volume of the output). + The optional ``offset`` and ``limit`` parameters can be used to specify the range of the versions list to be shown. This can be used to paginate through the list in a dataset with a large number of versions. @@ -1174,6 +1176,12 @@ The fully expanded example above (without environment variables) looks like this The optional ``excludeFiles`` parameter specifies whether the files should be listed in the output (defaults to ``true``). Note that a separate ``/files`` API can be used for listing the files, or a subset thereof in a given version. +.. code-block:: bash + + curl "https://demo.dataverse.org/api/datasets/24/versions/1.0?excludeMetadataBlocks=false" + +The optional ``excludeMetadataBlocks`` parameter specifies whether the metadataBlocks should be listed in the output (defaults to ``true``). + By default, deaccessioned dataset versions are not included in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a "not found" error if the version is deaccessioned and you do not enable the ``includeDeaccessioned`` option described below. diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 4b919c5ed82..d576022389c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -415,15 +415,16 @@ public Response useDefaultCitationDate(@Context ContainerRequestContext crc, @Pa @GET @AuthRequired @Path("{id}/versions") - public Response listVersions(@Context ContainerRequestContext crc, @PathParam("id") String id, @QueryParam("excludeFiles") Boolean excludeFiles, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset) { + public Response listVersions(@Context ContainerRequestContext crc, @PathParam("id") String id, @QueryParam("excludeFiles") Boolean excludeFiles,@QueryParam("excludeMetadataBlocks") Boolean excludeMetadataBlocks, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset) { return response( req -> { Dataset dataset = findDatasetOrDie(id); Boolean deepLookup = excludeFiles == null ? true : !excludeFiles; + Boolean includeMetadataBlocks = excludeMetadataBlocks == null ? true : !excludeMetadataBlocks; return ok( execCommand( new ListVersionsCommand(req, dataset, offset, limit, deepLookup) ) .stream() - .map( d -> json(d, deepLookup) ) + .map( d -> json(d, deepLookup, includeMetadataBlocks) ) .collect(toJsonArray())); }, getRequestUser(crc)); } @@ -435,6 +436,7 @@ public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @QueryParam("excludeFiles") Boolean excludeFiles, + @QueryParam("excludeMetadataBlocks") Boolean excludeMetadataBlocks, @QueryParam("includeDeaccessioned") boolean includeDeaccessioned, @QueryParam("returnOwners") boolean returnOwners, @Context UriInfo uriInfo, @@ -460,11 +462,12 @@ public Response getVersion(@Context ContainerRequestContext crc, if (excludeFiles == null ? true : !excludeFiles) { requestedDatasetVersion = datasetversionService.findDeep(requestedDatasetVersion.getId()); } + Boolean includeMetadataBlocks = excludeMetadataBlocks == null ? true : !excludeMetadataBlocks; JsonObjectBuilder jsonBuilder = json(requestedDatasetVersion, null, - excludeFiles == null ? true : !excludeFiles, - returnOwners); + excludeFiles == null ? true : !excludeFiles, + returnOwners, includeMetadataBlocks); return ok(jsonBuilder); }, getRequestUser(crc)); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index c908a4d2bce..d9c2250ef6f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -418,11 +418,17 @@ public static JsonObjectBuilder json(FileDetailsHolder ds) { } public static JsonObjectBuilder json(DatasetVersion dsv, boolean includeFiles) { - return json(dsv, null, includeFiles, false); + return json(dsv, null, includeFiles, false,true); + } + public static JsonObjectBuilder json(DatasetVersion dsv, boolean includeFiles, boolean includeMetadataBlocks) { + return json(dsv, null, includeFiles, false, includeMetadataBlocks); + } + public static JsonObjectBuilder json(DatasetVersion dsv, List anonymizedFieldTypeNamesList, + boolean includeFiles, boolean returnOwners) { + return json( dsv, anonymizedFieldTypeNamesList, includeFiles, returnOwners,true); } - public static JsonObjectBuilder json(DatasetVersion dsv, List anonymizedFieldTypeNamesList, - boolean includeFiles, boolean returnOwners) { + boolean includeFiles, boolean returnOwners, boolean includeMetadataBlocks) { Dataset dataset = dsv.getDataset(); JsonObjectBuilder bld = jsonObjectBuilder() .add("id", dsv.getId()).add("datasetId", dataset.getId()) @@ -467,11 +473,12 @@ public static JsonObjectBuilder json(DatasetVersion dsv, List anonymized .add("sizeOfCollection", dsv.getTermsOfUseAndAccess().getSizeOfCollection()) .add("studyCompletion", dsv.getTermsOfUseAndAccess().getStudyCompletion()) .add("fileAccessRequest", dsv.getTermsOfUseAndAccess().isFileAccessRequest()); - - bld.add("metadataBlocks", (anonymizedFieldTypeNamesList != null) ? - jsonByBlocks(dsv.getDatasetFields(), anonymizedFieldTypeNamesList) - : jsonByBlocks(dsv.getDatasetFields()) - ); + if(includeMetadataBlocks) { + bld.add("metadataBlocks", (anonymizedFieldTypeNamesList != null) ? + jsonByBlocks(dsv.getDatasetFields(), anonymizedFieldTypeNamesList) + : jsonByBlocks(dsv.getDatasetFields()) + ); + } if(returnOwners){ bld.add("isPartOf", getOwnersFromDvObject(dataset)); } From 408172c06a60d88af9e4c9c72b8f153867958779 Mon Sep 17 00:00:00 2001 From: darms Date: Mon, 19 Aug 2024 13:40:59 +0200 Subject: [PATCH 0036/1048] feat(api/versions): Added a new optional property to hide metadataBlocks from API response. --- doc/release-notes/10171-exlude-metadatablocks.md | 2 +- doc/sphinx-guides/source/api/native-api.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/release-notes/10171-exlude-metadatablocks.md b/doc/release-notes/10171-exlude-metadatablocks.md index 64fadfcc35e..8039b36617c 100644 --- a/doc/release-notes/10171-exlude-metadatablocks.md +++ b/doc/release-notes/10171-exlude-metadatablocks.md @@ -1,4 +1,4 @@ Extension of API `{id}/versions` and `{id}/versions/{versionId}` with an optional ``excludeMetadataBlocks`` parameter, -that specifies whether the metadataBlocks should be listed in the output. It defaults to ``true``, preserving backward +that specifies whether the metadataBlocks should be listed in the output. It defaults to ``false``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or metadataBlocks having the metadata blocks included can dramatically increase the volume of the output). diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 5168b8c5917..a3a004969dc 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1150,7 +1150,7 @@ It returns a list of versions with their metadata, and file list: The optional ``excludeFiles`` parameter specifies whether the files should be listed in the output. It defaults to ``true``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or files having the files included can dramatically increase the volume of the output). A separate ``/files`` API can be used for listing the files, or a subset thereof in a given version. -The optional ``excludeMetadataBlocks`` parameter specifies whether the metadataBlocks should be listed in the output. It defaults to ``true``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or metadataBlocks having the metadata blocks included can dramatically increase the volume of the output). +The optional ``excludeMetadataBlocks`` parameter specifies whether the metadataBlocks should be listed in the output. It defaults to ``false``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or metadataBlocks having the metadata blocks included can dramatically increase the volume of the output). The optional ``offset`` and ``limit`` parameters can be used to specify the range of the versions list to be shown. This can be used to paginate through the list in a dataset with a large number of versions. @@ -1180,7 +1180,7 @@ The optional ``excludeFiles`` parameter specifies whether the files should be li curl "https://demo.dataverse.org/api/datasets/24/versions/1.0?excludeMetadataBlocks=false" -The optional ``excludeMetadataBlocks`` parameter specifies whether the metadataBlocks should be listed in the output (defaults to ``true``). +The optional ``excludeMetadataBlocks`` parameter specifies whether the metadataBlocks should be listed in the output (defaults to ``false``). By default, deaccessioned dataset versions are not included in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a "not found" error if the version is deaccessioned and you do not enable the ``includeDeaccessioned`` option described below. From 2b2da9935c43d79580f84266b447a46b15858033 Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Thu, 22 Aug 2024 13:22:03 +0200 Subject: [PATCH 0037/1048] fix: remove unnecessary slash in perma PIDs in DDI exporters --- .../edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 9a689f7a4ed..cd7a9a40068 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -163,7 +163,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) String persistentAuthority = datasetDto.getAuthority(); String persistentId = datasetDto.getIdentifier(); - String pid = persistentProtocol + ":" + persistentAuthority + "/" + persistentId; + String pid = persistentProtocol + ":" + persistentAuthority + ("perma".equals(persistentAgency) ? "" : "/") + persistentId; String pidUri = pid; //Some tests don't send real PIDs - don't try to get their URL form if(!pidUri.equals("null:null/null")) { @@ -344,7 +344,7 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase writeFullElement(xmlw, "titl", dto2Primitive(version, DatasetFieldConstant.title), datasetDto.getMetadataLanguage()); xmlw.writeStartElement("IDNo"); writeAttribute(xmlw, "agency", persistentAgency); - xmlw.writeCharacters(persistentProtocol + ":" + persistentAuthority + "/" + persistentId); + xmlw.writeCharacters(persistentProtocol + ":" + persistentAuthority + ("perma".equals(persistentAgency) ? "" : "/") + persistentId); xmlw.writeEndElement(); // IDNo xmlw.writeEndElement(); // titlStmt xmlw.writeStartElement("distStmt"); From c8cdb8bf46b9c4a9a6836cab418d8012911689be Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Thu, 22 Aug 2024 13:22:59 +0200 Subject: [PATCH 0038/1048] fix: remove doi property for perma PIDs in BibTeX citation --- .../java/edu/harvard/iq/dataverse/DataCitation.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java index 3977023fc4b..60e9e10af99 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java @@ -293,11 +293,13 @@ public void writeAsBibtexCitation(OutputStream os) throws IOException { out.write("version = {"); out.write(version); out.write("},\r\n"); - out.write("doi = {"); - out.write(persistentId.getAuthority()); - out.write("/"); - out.write(persistentId.getIdentifier()); - out.write("},\r\n"); + if("doi".equals(persistentId.getProtocol())) { + out.write("doi = {"); + out.write(persistentId.getAuthority()); + out.write("/"); + out.write(persistentId.getIdentifier()); + out.write("},\r\n"); + } out.write("url = {"); out.write(persistentId.asURL()); out.write("}\r\n"); From ccf74f94255a41a4a3626ae552fa2a0c419cba81 Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Thu, 22 Aug 2024 13:23:40 +0200 Subject: [PATCH 0039/1048] fix: fix PID formatting in EndNote XML citation --- src/main/java/edu/harvard/iq/dataverse/DataCitation.java | 3 +-- src/test/java/edu/harvard/iq/dataverse/DataCitationTest.java | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java index 60e9e10af99..af5cb031cde 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java @@ -621,8 +621,7 @@ private void createEndNoteXML(XMLStreamWriter xmlw) throws XMLStreamException { } if (persistentId != null) { xmlw.writeStartElement("electronic-resource-num"); - String electResourceNum = persistentId.getProtocol() + "/" + persistentId.getAuthority() + "/" - + persistentId.getIdentifier(); + String electResourceNum = persistentId.asString(); xmlw.writeCharacters(electResourceNum); xmlw.writeEndElement(); } diff --git a/src/test/java/edu/harvard/iq/dataverse/DataCitationTest.java b/src/test/java/edu/harvard/iq/dataverse/DataCitationTest.java index 23a7efedca7..acaedac9e38 100644 --- a/src/test/java/edu/harvard/iq/dataverse/DataCitationTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/DataCitationTest.java @@ -266,7 +266,7 @@ public void testToEndNoteString_withTitleAndAuthor() throws ParseException { "V1" + "LibraScholar" + "https://doi.org/10.5072/FK2/LK0D1H" + - "doi/10.5072/FK2/LK0D1H" + + "doi:10.5072/FK2/LK0D1H" + "" + "" + ""; @@ -295,7 +295,7 @@ public void testToEndNoteString_withoutTitleAndAuthor() throws ParseException { "V1" + "LibraScholar" + "https://doi.org/10.5072/FK2/LK0D1H" + - "doi/10.5072/FK2/LK0D1H" + + "doi:10.5072/FK2/LK0D1H" + "" + "" + ""; From 34a795c97db8bcfce246bff51231946745981a05 Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Thu, 22 Aug 2024 18:00:56 +0200 Subject: [PATCH 0040/1048] fix: use correct PID separator in DDI exporters (WIP) --- .../java/edu/harvard/iq/dataverse/DvObject.java | 16 +++++++++++++++- .../java/edu/harvard/iq/dataverse/GlobalId.java | 2 ++ .../harvard/iq/dataverse/api/dto/DatasetDTO.java | 11 ++++++++++- .../iq/dataverse/export/ddi/DdiExportUtil.java | 6 ++++-- .../iq/dataverse/util/json/JsonPrinter.java | 1 + src/main/resources/db/migration/V6.3.0.2.sql | 1 + 6 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 src/main/resources/db/migration/V6.3.0.2.sql diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObject.java b/src/main/java/edu/harvard/iq/dataverse/DvObject.java index cc5d7620969..73033f7ab4c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DvObject.java +++ b/src/main/java/edu/harvard/iq/dataverse/DvObject.java @@ -144,12 +144,14 @@ public String visit(DataFile df) { @Column(insertable = false, updatable = false) private String dtype; /* - * Add DOI related fields + * Add PID related fields */ private String protocol; private String authority; + private String separator; + @Temporal(value = TemporalType.TIMESTAMP) private Date globalIdCreateTime; @@ -324,6 +326,16 @@ public void setAuthority(String authority) { globalId=null; } + public String getSeparator() { + return separator; + } + + public void setSeparator(String separator) { + this.separator = separator; + //Remove cached value + globalId=null; + } + public Date getGlobalIdCreateTime() { return globalIdCreateTime; } @@ -354,11 +366,13 @@ public void setGlobalId( GlobalId pid ) { if ( pid == null ) { setProtocol(null); setAuthority(null); + setSeparator(null); setIdentifier(null); } else { //These reset globalId=null setProtocol(pid.getProtocol()); setAuthority(pid.getAuthority()); + setSeparator(pid.getSeparator()); setIdentifier(pid.getIdentifier()); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/GlobalId.java b/src/main/java/edu/harvard/iq/dataverse/GlobalId.java index a542cb52ac0..531023a8e80 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GlobalId.java +++ b/src/main/java/edu/harvard/iq/dataverse/GlobalId.java @@ -63,6 +63,8 @@ public String getAuthority() { return authority; } + public String getSeparator() { return separator; } + public String getIdentifier() { return identifier; } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetDTO.java index 3fc31730ba2..ec8adfb4eef 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetDTO.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetDTO.java @@ -12,6 +12,7 @@ public class DatasetDTO implements java.io.Serializable { private String identifier; private String protocol; private String authority; + private String separator; private String globalIdCreateTime; private String publisher; private String publicationDate; @@ -51,6 +52,14 @@ public void setAuthority(String authority) { this.authority = authority; } + public String getSeparator() { + return separator; + } + + public void setSeparator(String separator) { + this.separator = separator; + } + public String getGlobalIdCreateTime() { return globalIdCreateTime; } @@ -94,7 +103,7 @@ public void setPublicationDate(String publicationDate) { @Override public String toString() { - return "DatasetDTO{" + "id=" + id + ", identifier=" + identifier + ", protocol=" + protocol + ", authority=" + authority + ", globalIdCreateTime=" + globalIdCreateTime + ", datasetVersion=" + datasetVersion + ", dataFiles=" + dataFiles + '}'; + return "DatasetDTO{" + "id=" + id + ", identifier=" + identifier + ", protocol=" + protocol + ", authority=" + authority + ", separator=" + separator + ", globalIdCreateTime=" + globalIdCreateTime + ", datasetVersion=" + datasetVersion + ", dataFiles=" + dataFiles + '}'; } public void setMetadataLanguage(String metadataLanguage) { diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index cd7a9a40068..d4e9e700b9b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -162,8 +162,9 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) String persistentAuthority = datasetDto.getAuthority(); String persistentId = datasetDto.getIdentifier(); + String persistentSeparator = datasetDto.getSeparator(); - String pid = persistentProtocol + ":" + persistentAuthority + ("perma".equals(persistentAgency) ? "" : "/") + persistentId; + String pid = persistentProtocol + ":" + persistentAuthority + persistentSeparator + persistentId; String pidUri = pid; //Some tests don't send real PIDs - don't try to get their URL form if(!pidUri.equals("null:null/null")) { @@ -337,6 +338,7 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase String persistentAuthority = datasetDto.getAuthority(); String persistentId = datasetDto.getIdentifier(); + String persistentSeparator = datasetDto.getSeparator(); xmlw.writeStartElement("docDscr"); xmlw.writeStartElement("citation"); @@ -344,7 +346,7 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase writeFullElement(xmlw, "titl", dto2Primitive(version, DatasetFieldConstant.title), datasetDto.getMetadataLanguage()); xmlw.writeStartElement("IDNo"); writeAttribute(xmlw, "agency", persistentAgency); - xmlw.writeCharacters(persistentProtocol + ":" + persistentAuthority + ("perma".equals(persistentAgency) ? "" : "/") + persistentId); + xmlw.writeCharacters(persistentProtocol + ":" + persistentAuthority + persistentSeparator + persistentId); xmlw.writeEndElement(); // IDNo xmlw.writeEndElement(); // titlStmt xmlw.writeStartElement("distStmt"); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index adb7cf98975..81409bb5ea4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -397,6 +397,7 @@ public static JsonObjectBuilder json(Dataset ds, Boolean returnOwners) { .add("persistentUrl", ds.getPersistentURL()) .add("protocol", ds.getProtocol()) .add("authority", ds.getAuthority()) + .add("separator", ds.getSeparator()) .add("publisher", BrandingUtil.getInstallationBrandName()) .add("publicationDate", ds.getPublicationDateFormattedYYYYMMDD()) .add("storageIdentifier", ds.getStorageIdentifier()); diff --git a/src/main/resources/db/migration/V6.3.0.2.sql b/src/main/resources/db/migration/V6.3.0.2.sql new file mode 100644 index 00000000000..60118d220a4 --- /dev/null +++ b/src/main/resources/db/migration/V6.3.0.2.sql @@ -0,0 +1 @@ +ALTER TABLE dvobject ADD COLUMN IF NOT EXISTS separator character varying(255); \ No newline at end of file From 825d5033c1f9e0a45d628f2aa3d9e20f315fcc1e Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Fri, 23 Aug 2024 10:29:08 +0200 Subject: [PATCH 0041/1048] fix: use correct PID separator in DDI exporters (WIP) --- src/main/java/edu/harvard/iq/dataverse/GlobalId.java | 4 +++- .../edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java | 2 +- .../export/ddi/dataset-create-new-all-ddi-fields.json | 1 + .../edu/harvard/iq/dataverse/export/ddi/dataset-finch1.json | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/GlobalId.java b/src/main/java/edu/harvard/iq/dataverse/GlobalId.java index 531023a8e80..cd1c00810cc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GlobalId.java +++ b/src/main/java/edu/harvard/iq/dataverse/GlobalId.java @@ -63,7 +63,9 @@ public String getAuthority() { return authority; } - public String getSeparator() { return separator; } + public String getSeparator() { + return separator; + } public String getIdentifier() { return identifier; diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index d4e9e700b9b..b4cfb373fcc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -167,7 +167,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) String pid = persistentProtocol + ":" + persistentAuthority + persistentSeparator + persistentId; String pidUri = pid; //Some tests don't send real PIDs - don't try to get their URL form - if(!pidUri.equals("null:null/null")) { + if(!pidUri.equals("null:nullnullnull")) { pidUri= PidUtil.parseAsGlobalID(persistentProtocol, persistentAuthority, persistentId).asURL(); } // The "persistentAgency" tag is used for the "agency" attribute of the diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json index 9cf04bd0e05..b0dace0fb86 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json @@ -4,6 +4,7 @@ "persistentUrl": "https://doi.org/10.5072/FK2/WKUKGV", "protocol": "doi", "authority": "10.5072/FK2", + "separator": "/", "publisher": "Root", "publicationDate": "2020-02-19", "datasetVersion": { diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.json b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.json index 2d4ca078962..49565d925ab 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.json +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.json @@ -4,6 +4,7 @@ "persistentUrl": "https://doi.org/10.5072/FK2/PCA2E3", "protocol": "doi", "authority": "10.5072/FK2", + "separator": "/", "metadataLanguage": "en", "datasetVersion": { "id": 2, From b8a068e3a8defc49bc3ff12fdc8c650dc1fb44df Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Fri, 23 Aug 2024 12:10:13 +0200 Subject: [PATCH 0042/1048] fix: use correct PID separator in DDI exporters --- src/main/resources/db/migration/V6.3.0.2.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V6.3.0.2.sql b/src/main/resources/db/migration/V6.3.0.2.sql index 60118d220a4..9c3b24712e1 100644 --- a/src/main/resources/db/migration/V6.3.0.2.sql +++ b/src/main/resources/db/migration/V6.3.0.2.sql @@ -1 +1,3 @@ -ALTER TABLE dvobject ADD COLUMN IF NOT EXISTS separator character varying(255); \ No newline at end of file +ALTER TABLE dvobject ADD COLUMN IF NOT EXISTS separator character varying(255) DEFAULT ''; + +UPDATE dvobject SET separator='/' WHERE protocol = 'doi' OR protocol = 'hdl'; \ No newline at end of file From 70031f35fb834e81b2c12734a48204748e102d67 Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Mon, 26 Aug 2024 14:40:36 +0200 Subject: [PATCH 0043/1048] fix: use GlobalId methods to create PID URLs/strings in DdiExportUtil --- .../dataverse/export/ddi/DdiExportUtil.java | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index b4cfb373fcc..20eb67349f3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -5,6 +5,7 @@ import edu.harvard.iq.dataverse.ControlledVocabularyValue; import edu.harvard.iq.dataverse.DatasetFieldConstant; import edu.harvard.iq.dataverse.DvObjectContainer; +import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.api.dto.DatasetDTO; import edu.harvard.iq.dataverse.api.dto.DatasetVersionDTO; import edu.harvard.iq.dataverse.api.dto.FieldDTO; @@ -79,6 +80,10 @@ public class DdiExportUtil { public static final String NOTE_SUBJECT_CONTENTTYPE = "Content/MIME Type"; public static final String CITATION_BLOCK_NAME = "citation"; + //Some tests don't send real PIDs that can be parsed + //Use constant empty PID in these cases + private static final String EMPTY_PID = "null:nullnullnull"; + public static String datasetDtoAsJson2ddi(String datasetDtoAsJson) { Gson gson = new Gson(); DatasetDTO datasetDto = gson.fromJson(datasetDtoAsJson, DatasetDTO.class); @@ -162,13 +167,15 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) String persistentAuthority = datasetDto.getAuthority(); String persistentId = datasetDto.getIdentifier(); - String persistentSeparator = datasetDto.getSeparator(); - String pid = persistentProtocol + ":" + persistentAuthority + persistentSeparator + persistentId; - String pidUri = pid; - //Some tests don't send real PIDs - don't try to get their URL form - if(!pidUri.equals("null:nullnullnull")) { - pidUri= PidUtil.parseAsGlobalID(persistentProtocol, persistentAuthority, persistentId).asURL(); + GlobalId pid = PidUtil.parseAsGlobalID(persistentProtocol, persistentAuthority, persistentId); + String pidUri, pidString; + if(pid != null) { + pidUri = pid.asURL(); + pidString = pid.asString(); + } else { + pidUri = EMPTY_PID; + pidString = EMPTY_PID; } // The "persistentAgency" tag is used for the "agency" attribute of the // ddi section; back in the DVN3 days we used "handle" and "DOI" @@ -198,7 +205,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) writeAttribute(xmlw, "agency", persistentAgency); - xmlw.writeCharacters(pid); + xmlw.writeCharacters(pidString); xmlw.writeEndElement(); // IDNo writeOtherIdElement(xmlw, version); xmlw.writeEndElement(); // titlStmt @@ -338,15 +345,21 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase String persistentAuthority = datasetDto.getAuthority(); String persistentId = datasetDto.getIdentifier(); - String persistentSeparator = datasetDto.getSeparator(); - + GlobalId pid = PidUtil.parseAsGlobalID(persistentProtocol, persistentAuthority, persistentId); + String pidString; + if(pid != null) { + pidString = pid.asString(); + } else { + pidString = EMPTY_PID; + } + xmlw.writeStartElement("docDscr"); xmlw.writeStartElement("citation"); xmlw.writeStartElement("titlStmt"); writeFullElement(xmlw, "titl", dto2Primitive(version, DatasetFieldConstant.title), datasetDto.getMetadataLanguage()); xmlw.writeStartElement("IDNo"); writeAttribute(xmlw, "agency", persistentAgency); - xmlw.writeCharacters(persistentProtocol + ":" + persistentAuthority + persistentSeparator + persistentId); + xmlw.writeCharacters(pidString); xmlw.writeEndElement(); // IDNo xmlw.writeEndElement(); // titlStmt xmlw.writeStartElement("distStmt"); From 7884bdb7aac11eababf2169bad2c98224dadfb50 Mon Sep 17 00:00:00 2001 From: Florian Fritze Date: Tue, 27 Aug 2024 10:19:56 +0200 Subject: [PATCH 0044/1048] added integration test: checking child / parent logic --- .../edu/harvard/iq/dataverse/api/DataversesIT.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index d682e4ade98..538d6492305 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -852,6 +852,16 @@ public void testListMetadataBlocks() { .body("data[0].displayName", equalTo("Citation Metadata")) .body("data[0].fields", not(equalTo(null))) .body("data.size()", equalTo(1)); + + // Checking child / parent logic + listMetadataBlocksResponse = UtilIT.getMetadataBlock("citation"); + listMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode()); + listMetadataBlocksResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].displayName", equalTo("Citation Metadata")) + .body("data[0].fields", not(equalTo(null))) + .body("data[0].fields.otherIdAgency", equalTo(null)) + .body("data[0].fields.otherId.childFields.size()", equalTo(2)); } @Test From f6e1b06974f9653f5b46b83e2a01777ebcdce0c1 Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Tue, 27 Aug 2024 10:32:59 +0200 Subject: [PATCH 0045/1048] fix: set separator when creating new dataset --- .../iq/dataverse/pidproviders/AbstractPidProvider.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java index f6d142aac96..03f6fc29697 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java @@ -202,6 +202,16 @@ public DvObject generatePid(DvObject dvObject) { + ") doesn't match that of the provider, id: " + getId()); } } + if (dvObject.getSeparator() == null) { + dvObject.setSeparator(getSeparator()); + } else { + if (!dvObject.getSeparator().equals(getSeparator())) { + logger.warning("The separator of the DvObject (" + dvObject.getSeparator() + + ") does not match the configured separator (" + getSeparator() + ")"); + throw new IllegalArgumentException("The separator of the DvObject (" + dvObject.getSeparator() + + ") doesn't match that of the provider, id: " + getId()); + } + } if (dvObject.isInstanceofDataset()) { dvObject.setIdentifier(generateDatasetIdentifier((Dataset) dvObject)); } else { From 46de8fbba52dcd8474c9efede96ef0fde65b8f09 Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Tue, 27 Aug 2024 13:10:53 +0200 Subject: [PATCH 0046/1048] test: add tests for DDI exports for datasets with PermaLinks --- .../export/ddi/DdiExportUtilTest.java | 93 +++++++++++++++++++ .../export/ddi/dataset-perma-w-separator.json | 92 ++++++++++++++++++ .../export/ddi/dataset-perma-w-separator.xml | 50 ++++++++++ .../dataverse/export/ddi/dataset-perma.json | 92 ++++++++++++++++++ .../iq/dataverse/export/ddi/dataset-perma.xml | 50 ++++++++++ 5 files changed, 377 insertions(+) create mode 100644 src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma-w-separator.json create mode 100644 src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma-w-separator.xml create mode 100644 src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma.json create mode 100644 src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma.xml diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java index 41e6be61bb8..b2c1e10bbf8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java @@ -1,6 +1,15 @@ package edu.harvard.iq.dataverse.export.ddi; +import edu.harvard.iq.dataverse.pidproviders.PidProviderFactory; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; +import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DataCiteDOIProvider; +import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DataCiteProviderFactory; +import edu.harvard.iq.dataverse.pidproviders.perma.PermaLinkPidProvider; +import edu.harvard.iq.dataverse.pidproviders.perma.PermaLinkProviderFactory; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.testing.JvmSetting; +import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; import edu.harvard.iq.dataverse.util.xml.XmlPrinter; import java.io.ByteArrayOutputStream; @@ -11,12 +20,17 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.logging.Logger; import edu.harvard.iq.dataverse.util.xml.html.HtmlPrinter; import org.jsoup.Jsoup; import org.jsoup.helper.W3CDom; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -33,6 +47,30 @@ import static org.junit.jupiter.api.Assertions.*; @ExtendWith(MockitoExtension.class) +@LocalJvmSettings +//Perma 1 +@JvmSetting(key = JvmSettings.PID_PROVIDER_LABEL, value = "perma 1", varArgs = "perma1") +@JvmSetting(key = JvmSettings.PID_PROVIDER_TYPE, value = PermaLinkPidProvider.TYPE, varArgs = "perma1") +@JvmSetting(key = JvmSettings.PID_PROVIDER_AUTHORITY, value = "PERM", varArgs = "perma1") +@JvmSetting(key = JvmSettings.PERMALINK_BASE_URL, value = "https://example.org", varArgs = "perma1") +//Perma 2 +@JvmSetting(key = JvmSettings.PID_PROVIDER_LABEL, value = "perma 2", varArgs = "perma2") +@JvmSetting(key = JvmSettings.PID_PROVIDER_TYPE, value = PermaLinkPidProvider.TYPE, varArgs = "perma2") +@JvmSetting(key = JvmSettings.PID_PROVIDER_AUTHORITY, value = "PERM2", varArgs = "perma2") +@JvmSetting(key = JvmSettings.PERMALINK_SEPARATOR, value = "-", varArgs = "perma2") +@JvmSetting(key = JvmSettings.PERMALINK_BASE_URL, value = "https://example.org", varArgs = "perma2") +// Datacite 1 +@JvmSetting(key = JvmSettings.PID_PROVIDER_LABEL, value = "dataCite 1", varArgs = "dc1") +@JvmSetting(key = JvmSettings.PID_PROVIDER_TYPE, value = DataCiteDOIProvider.TYPE, varArgs = "dc1") +@JvmSetting(key = JvmSettings.PID_PROVIDER_AUTHORITY, value = "10.5072", varArgs = "dc1") +@JvmSetting(key = JvmSettings.PID_PROVIDER_SHOULDER, value = "FK2", varArgs = "dc1") +@JvmSetting(key = JvmSettings.DATACITE_MDS_API_URL, value = "https://mds.test.datacite.org/", varArgs = "dc1") +@JvmSetting(key = JvmSettings.DATACITE_REST_API_URL, value = "https://api.test.datacite.org", varArgs ="dc1") +@JvmSetting(key = JvmSettings.DATACITE_USERNAME, value = "test", varArgs ="dc1") +@JvmSetting(key = JvmSettings.DATACITE_PASSWORD, value = "changeme", varArgs ="dc1") + +//List to instantiate +@JvmSetting(key = JvmSettings.PID_PROVIDERS, value = "perma1, perma2, dc1") public class DdiExportUtilTest { private static final Logger logger = Logger.getLogger(DdiExportUtilTest.class.getCanonicalName()); @@ -45,6 +83,25 @@ void setup() { Mockito.lenient().when(settingsSvc.isTrueForKey(SettingsServiceBean.Key.ExportInstallationAsDistributorOnlyWhenNotSet, false)).thenReturn(false); DdiExportUtil.injectSettingsService(settingsSvc); } + + @BeforeAll + public static void setUpClass() throws Exception { + Map pidProviderFactoryMap = new HashMap<>(); + pidProviderFactoryMap.put(PermaLinkPidProvider.TYPE, new PermaLinkProviderFactory()); + pidProviderFactoryMap.put(DataCiteDOIProvider.TYPE, new DataCiteProviderFactory()); + + PidUtil.clearPidProviders(); + + //Read list of providers to add + List providers = Arrays.asList(JvmSettings.PID_PROVIDERS.lookup().split(",\\s")); + //Iterate through the list of providers and add them using the PidProviderFactory of the appropriate type + for (String providerId : providers) { + System.out.println("Loading provider: " + providerId); + String type = JvmSettings.PID_PROVIDER_TYPE.lookup(providerId); + PidProviderFactory factory = pidProviderFactoryMap.get(type); + PidUtil.addToProviderList(factory.createPidProvider(providerId)); + } + } @Test @@ -64,6 +121,42 @@ public void testJson2DdiNoFiles() throws Exception { XmlAssert.assertThat(result).and(datasetAsDdi).ignoreWhitespace().areSimilar(); } + + @Test + public void testJson2DdiPermaLink() throws Exception { + // given + Path datasetVersionJson = Path.of("src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma.json"); + String datasetVersionAsJson = Files.readString(datasetVersionJson, StandardCharsets.UTF_8); + Path ddiFile = Path.of("src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma.xml"); + String datasetAsDdi = XmlPrinter.prettyPrintXml(Files.readString(ddiFile, StandardCharsets.UTF_8)); + logger.fine(datasetAsDdi); + + // when + String result = DdiExportUtil.datasetDtoAsJson2ddi(datasetVersionAsJson); + logger.fine(result); + + // then + XmlAssert.assertThat(result).and(datasetAsDdi).ignoreWhitespace().areSimilar(); + } + + + @Test + public void testJson2DdiPermaLinkWithSeparator() throws Exception { + // given + Path datasetVersionJson = Path.of("src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma-w-separator.json"); + String datasetVersionAsJson = Files.readString(datasetVersionJson, StandardCharsets.UTF_8); + Path ddiFile = Path.of("src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma-w-separator.xml"); + String datasetAsDdi = XmlPrinter.prettyPrintXml(Files.readString(ddiFile, StandardCharsets.UTF_8)); + logger.fine(datasetAsDdi); + + // when + String result = DdiExportUtil.datasetDtoAsJson2ddi(datasetVersionAsJson); + logger.fine(result); + + // then + XmlAssert.assertThat(result).and(datasetAsDdi).ignoreWhitespace().areSimilar(); + } + @Test public void testExportDDI() throws Exception { // given diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma-w-separator.json b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma-w-separator.json new file mode 100644 index 00000000000..9b51dc2ff91 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma-w-separator.json @@ -0,0 +1,92 @@ +{ + "id": 10, + "identifier": "123456789", + "persistentUrl": "https://example.org/citation?persistentId=PERM2-123456789", + "protocol": "perma", + "authority": "PERM2", + "separator": "-", + "datasetVersion": { + "id": 1, + "versionNumber": 1, + "versionMinorNumber": 0, + "versionState": "RELEASED", + "productionDate": "Production Date", + "lastUpdateTime": "2015-09-29T17:47:35Z", + "releaseTime": "2015-09-29T17:47:35Z", + "createTime": "2015-09-24T16:47:50Z", + "metadataBlocks": { + "citation": { + "displayName": "Citation Metadata", + "fields": [ + { + "typeName": "title", + "multiple": false, + "typeClass": "primitive", + "value": "Spruce Goose" + }, + { + "typeName": "author", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "authorName": { + "typeName": "authorName", + "multiple": false, + "typeClass": "primitive", + "value": "Spruce, Sabrina" + } + } + ] + }, + { + "typeName": "datasetContact", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "datasetContactEmail": { + "typeName": "datasetContactEmail", + "multiple": false, + "typeClass": "primitive", + "value": "spruce@mailinator.com" + } + } + ] + }, + { + "typeName": "dsDescription", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "dsDescriptionValue": { + "typeName": "dsDescriptionValue", + "multiple": false, + "typeClass": "primitive", + "value": "What the Spruce Goose was really made of." + } + } + ] + }, + { + "typeName": "subject", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ + "Other" + ] + }, + { + "typeName": "depositor", + "multiple": false, + "typeClass": "primitive", + "value": "Spruce, Sabrina" + } + ] + } + }, + "files": [], + "citation": "Spruce, Sabrina, 2015, \"Spruce Goose\", https://example.org/citation?persistentId=PERM2-123456789, Root Dataverse, V1" + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma-w-separator.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma-w-separator.xml new file mode 100644 index 00000000000..2a7d4d09846 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma-w-separator.xml @@ -0,0 +1,50 @@ + + + + + + Spruce Goose + perma:PERM2-123456789 + + + + 1 + + Spruce, Sabrina, 2015, "Spruce Goose", https://example.org/citation?persistentId=PERM2-123456789, Root Dataverse, V1 + + + + + + Spruce Goose + perma:PERM2-123456789 + + + Spruce, Sabrina + + + + Spruce, Sabrina + + + + + + Other + + What the Spruce Goose was really made of. + + + + + + + + + + + + + + + diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma.json b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma.json new file mode 100644 index 00000000000..eb8fc6d1d88 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma.json @@ -0,0 +1,92 @@ +{ + "id": 10, + "identifier": "123456789", + "persistentUrl": "https://example.org/citation?persistentId=PERM123456789", + "protocol": "perma", + "authority": "PERM", + "separator": "", + "datasetVersion": { + "id": 1, + "versionNumber": 1, + "versionMinorNumber": 0, + "versionState": "RELEASED", + "productionDate": "Production Date", + "lastUpdateTime": "2015-09-29T17:47:35Z", + "releaseTime": "2015-09-29T17:47:35Z", + "createTime": "2015-09-24T16:47:50Z", + "metadataBlocks": { + "citation": { + "displayName": "Citation Metadata", + "fields": [ + { + "typeName": "title", + "multiple": false, + "typeClass": "primitive", + "value": "Spruce Goose" + }, + { + "typeName": "author", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "authorName": { + "typeName": "authorName", + "multiple": false, + "typeClass": "primitive", + "value": "Spruce, Sabrina" + } + } + ] + }, + { + "typeName": "datasetContact", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "datasetContactEmail": { + "typeName": "datasetContactEmail", + "multiple": false, + "typeClass": "primitive", + "value": "spruce@mailinator.com" + } + } + ] + }, + { + "typeName": "dsDescription", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "dsDescriptionValue": { + "typeName": "dsDescriptionValue", + "multiple": false, + "typeClass": "primitive", + "value": "What the Spruce Goose was really made of." + } + } + ] + }, + { + "typeName": "subject", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ + "Other" + ] + }, + { + "typeName": "depositor", + "multiple": false, + "typeClass": "primitive", + "value": "Spruce, Sabrina" + } + ] + } + }, + "files": [], + "citation": "Spruce, Sabrina, 2015, \"Spruce Goose\", https://example.org/citation?persistentId=PERM123456789, Root Dataverse, V1" + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma.xml new file mode 100644 index 00000000000..341cb7435bd --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-perma.xml @@ -0,0 +1,50 @@ + + + + + + Spruce Goose + perma:PERM123456789 + + + + 1 + + Spruce, Sabrina, 2015, "Spruce Goose", https://example.org/citation?persistentId=PERM123456789, Root Dataverse, V1 + + + + + + Spruce Goose + perma:PERM123456789 + + + Spruce, Sabrina + + + + Spruce, Sabrina + + + + + + Other + + What the Spruce Goose was really made of. + + + + + + + + + + + + + + + From 76053453a21f3ff8853f15cde63f85f5fdff91d0 Mon Sep 17 00:00:00 2001 From: Florian Fritze Date: Wed, 28 Aug 2024 14:50:06 +0200 Subject: [PATCH 0047/1048] integration test fix --- .../iq/dataverse/api/DataversesIT.java | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 538d6492305..a3d52c28a05 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -776,17 +776,23 @@ public void testListMetadataBlocks() { // Since the included property of notesText is set to false, we should retrieve the total number of fields minus one int citationMetadataBlockIndex = geospatialMetadataBlockIndex == 0 ? 1 : 0; listMetadataBlocksResponse.then().assertThat() - .body(String.format("data[%d].fields.size()", citationMetadataBlockIndex), equalTo(78)); + .body(String.format("data[%d].fields.size()", citationMetadataBlockIndex), equalTo(34)); // Since the included property of geographicCoverage is set to false, we should retrieve the total number of fields minus one listMetadataBlocksResponse.then().assertThat() - .body(String.format("data[%d].fields.size()", geospatialMetadataBlockIndex), equalTo(10)); + .body(String.format("data[%d].fields.size()", geospatialMetadataBlockIndex), equalTo(2)); + + listMetadataBlocksResponse = UtilIT.getMetadataBlock("geospatial"); - String actualGeospatialMetadataField1 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.geographicCoverage.name", geospatialMetadataBlockIndex)); - String actualGeospatialMetadataField2 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.country.name", geospatialMetadataBlockIndex)); - String actualGeospatialMetadataField3 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.city.name", geospatialMetadataBlockIndex)); + String actualGeospatialMetadataField1 = listMetadataBlocksResponse.then().extract().path(String.format("data.fields['geographicCoverage'].name")); + String actualGeospatialMetadataField2 = listMetadataBlocksResponse.then().extract().path(String.format("data.fields['geographicCoverage'].childFields['country'].name")); + String actualGeospatialMetadataField3 = listMetadataBlocksResponse.then().extract().path(String.format("data.fields['geographicCoverage'].childFields['city'].name")); + + listMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode()) + .body("data.fields['geographicCoverage'].childFields.size()", equalTo(4)) + .body("data.fields['geographicBoundingBox'].childFields.size()", equalTo(4)); - assertNull(actualGeospatialMetadataField1); + assertNotNull(actualGeospatialMetadataField1); assertNotNull(actualGeospatialMetadataField2); assertNotNull(actualGeospatialMetadataField3); @@ -809,21 +815,21 @@ public void testListMetadataBlocks() { geospatialMetadataBlockIndex = actualMetadataBlockDisplayName2.equals("Geospatial Metadata") ? 1 : 0; listMetadataBlocksResponse.then().assertThat() - .body(String.format("data[%d].fields.size()", geospatialMetadataBlockIndex), equalTo(1)); + .body(String.format("data[%d].fields.size()", geospatialMetadataBlockIndex), equalTo(0)); - actualGeospatialMetadataField1 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.geographicCoverage.name", geospatialMetadataBlockIndex)); - actualGeospatialMetadataField2 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.country.name", geospatialMetadataBlockIndex)); - actualGeospatialMetadataField3 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.city.name", geospatialMetadataBlockIndex)); +// actualGeospatialMetadataField1 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.geographicCoverage.name", geospatialMetadataBlockIndex)); +// actualGeospatialMetadataField2 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.geographicCoverage.childFields['country'].name", geospatialMetadataBlockIndex)); +// actualGeospatialMetadataField3 = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.geographicCoverage.childFields['city'].name", geospatialMetadataBlockIndex)); - assertNull(actualGeospatialMetadataField1); - assertNotNull(actualGeospatialMetadataField2); - assertNull(actualGeospatialMetadataField3); +// assertNull(actualGeospatialMetadataField1); +// assertNotNull(actualGeospatialMetadataField2); +// assertNull(actualGeospatialMetadataField3); citationMetadataBlockIndex = geospatialMetadataBlockIndex == 0 ? 1 : 0; // notesText has displayOnCreate=true but has include=false, so should not be retrieved String notesTextCitationMetadataField = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.notesText.name", citationMetadataBlockIndex)); - assertNull(notesTextCitationMetadataField); + assertNotNull(notesTextCitationMetadataField); // producerName is a conditionally required field, so should not be retrieved String producerNameCitationMetadataField = listMetadataBlocksResponse.then().extract().path(String.format("data[%d].fields.producerName.name", citationMetadataBlockIndex)); @@ -858,10 +864,10 @@ public void testListMetadataBlocks() { listMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode()); listMetadataBlocksResponse.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data[0].displayName", equalTo("Citation Metadata")) - .body("data[0].fields", not(equalTo(null))) - .body("data[0].fields.otherIdAgency", equalTo(null)) - .body("data[0].fields.otherId.childFields.size()", equalTo(2)); + .body("data.displayName", equalTo("Citation Metadata")) + .body("data.fields", not(equalTo(null))) + .body("data.fields.otherIdAgency", equalTo(null)) + .body("data.fields.otherId.childFields.size()", equalTo(2)); } @Test From 51ebe21fbd203526cb4afa826f15ffcefa796f28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asbj=C3=B8rn=20Sk=C3=B8dt?= Date: Fri, 30 Aug 2024 10:54:53 +0200 Subject: [PATCH 0048/1048] Update archival.tsv --- scripts/api/data/metadatablocks/archival.tsv | 31 ++++++++++---------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/scripts/api/data/metadatablocks/archival.tsv b/scripts/api/data/metadatablocks/archival.tsv index 07fa14fdc9e..2b472acfc67 100644 --- a/scripts/api/data/metadatablocks/archival.tsv +++ b/scripts/api/data/metadatablocks/archival.tsv @@ -1,19 +1,20 @@ #metadataBlock name dataverseAlias displayName blockURI archival Archival Metadata #datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id termURI - submitToArchivalAppraisal Archival Appraisal An assessment whether the dataset should be submitted for archival appraisal text 0 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE archive - archivedFrom Archived from A date (YYYY-MM-DD) from which the dataset is archived YYYY-MM-DD date 2 #VALUE TRUE FALSE FALSE FALSE FALSE FALSE archive - archivedFor Retention period The retention period for which the archived dataset is to be kept by the holding archive text 3 #VALUE TRUE TRUE FALSE FALSE FALSE FALSE archive - holdingArchive Archived at Holding Archive The holding archive where the dataset is archived text 4 #VALUE FALSE FALSE TRUE FALSE FALSE FALSE archive https://schema.org/holdingArchive - archivedAt Archived at URL URL to holding archive where the dataset is archived URL url 5 #VALUE FALSE FALSE TRUE FALSE FALSE FALSE archive https://schema.org/archivedAt + submitToArchivalAppraisal Submit to Archival Appraisal Your assessment whether the dataset should be submitted for archival appraisal text 0 #VALUE TRUE TRUE FALSE FALSE TRUE FALSE archival + archivedFrom Archived from A date (YYYY-MM-DD) from which the dataset is archived YYYY-MM-DD date 1 #VALUE TRUE FALSE FALSE FALSE FALSE FALSE archival + archivedFor Retention period The retention period for which the archived dataset is to be kept by the holding archive text 2 #VALUE TRUE TRUE FALSE FALSE FALSE FALSE archival + holdingArchive Holding Archive Information on the holding archive where the dataset is archived none 3 FALSE FALSE TRUE FALSE FALSE FALSE archival + holdingArchiveName Archived at Holding Archive The name of the holding archive text 4 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE holdingArchive archival https://schema.org/holdingArchive + archivedAt Archived at URL URL to holding archive URL url 5 "#VALUE" FALSE FALSE FALSE FALSE FALSE FALSE holdingArchive archival https://schema.org/archivedAt #controlledVocabulary DatasetField Value identifier displayOrder - archivedFor 1 year 1_year 0 - archivedFor 3 years 3_years 1 - archivedFor 5 years 5_years 2 - archivedFor 10 years 10_years 3 - archivedFor 20 years 20_years 4 - archivedFor Indefinitely indefinitely 5 - archivedFor Unknown unknown 6 - submitToArchivalAppraisal Yes yes 0 - submitToArchivalAppraisal No no 1 - submitToArchivalAppraisal Unknown unknown 2 + archivedFor 1 year 0 + archivedFor 3 years 1 + archivedFor 5 years 2 + archivedFor 10 years 3 + archivedFor 20 years 4 + archivedFor Indefinitely 5 + archivedFor Unknown 6 + submitToArchivalAppraisal Yes 0 + submitToArchivalAppraisal No 1 + submitToArchivalAppraisal Unknown 2 From 022b862e04f714b8ff9aed2046cea2277107173f Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Thu, 5 Sep 2024 15:30:28 +0200 Subject: [PATCH 0049/1048] Add missing imports after merge --- .../java/edu/harvard/iq/dataverse/search/IndexServiceBean.java | 1 + .../edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index cfdb20c7e6b..168517c0ebf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -16,6 +16,7 @@ import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; +import edu.harvard.iq.dataverse.DatasetVersionServiceBean; import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DataverseLinkingServiceBean; import edu.harvard.iq.dataverse.DataverseServiceBean; diff --git a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java index f9373f7dd78..7af466021de 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java @@ -9,6 +9,7 @@ import edu.harvard.iq.dataverse.DatasetFieldType; import edu.harvard.iq.dataverse.DatasetFieldValue; import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.DatasetVersionServiceBean; import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.Dataverse.DataverseType; import edu.harvard.iq.dataverse.DataverseServiceBean; From 89549d7fd482a463b9574752cf6908fe4bd36a98 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 16 Sep 2024 10:13:02 -0400 Subject: [PATCH 0050/1048] fix typo #8808 #10575 --- .../iq/dataverse/engine/command/impl/CreateRoleCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommand.java index 08923125bdf..4a897adefa2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommand.java @@ -47,7 +47,7 @@ public DataverseRole execute(CommandContext ctxt) throws CommandException { throw new IllegalCommandException(BundleUtil.getStringFromBundle("permission.role.not.created.alias.already.exists"), this); } } catch (NoResultException nre) { - // we want no results because that meand we can create a role + // we want no results because that meant we can create a role } dv.addRole(role); return ctxt.roles().save(role); From fa804197bbae4d6da4f32167eff4a298546cf05b Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Fri, 27 Sep 2024 13:54:53 +0200 Subject: [PATCH 0051/1048] feat: index numerical and date fields in Solr with appropriate types --- conf/solr/schema.xml | 2 ++ .../java/edu/harvard/iq/dataverse/DatasetFieldType.java | 9 ++++----- .../harvard/iq/dataverse/search/IndexServiceBean.java | 4 +++- .../harvard/iq/dataverse/search/SearchServiceBean.java | 2 +- .../java/edu/harvard/iq/dataverse/search/SolrField.java | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/conf/solr/schema.xml b/conf/solr/schema.xml index 2aed50e9998..02e699722f7 100644 --- a/conf/solr/schema.xml +++ b/conf/solr/schema.xml @@ -814,6 +814,8 @@ + + diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java index 01785359e0e..2c385268fa5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java @@ -531,15 +531,14 @@ public String getDisplayName() { public SolrField getSolrField() { SolrField.SolrType solrType = SolrField.SolrType.TEXT_EN; if (fieldType != null) { - - /** - * @todo made more decisions based on fieldType: index as dates, - * integers, and floats so we can do range queries etc. - */ if (fieldType.equals(FieldType.DATE)) { solrType = SolrField.SolrType.DATE; } else if (fieldType.equals(FieldType.EMAIL)) { solrType = SolrField.SolrType.EMAIL; + } else if (fieldType.equals(FieldType.INT)) { + solrType = SolrField.SolrType.INTEGER; + } else if (fieldType.equals(FieldType.FLOAT)) { + solrType = SolrField.SolrType.FLOAT; } Boolean parentAllowsMultiplesBoolean = false; diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index a8cf9ed519b..e73b8d2f679 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1061,6 +1061,8 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set datasetFields = datasetFieldService.findAllOrderedById(); Map solrFieldsToHightlightOnMap = new HashMap<>(); if (addHighlights) { - solrQuery.setHighlight(true).setHighlightSnippets(1); + solrQuery.setHighlight(true).setHighlightSnippets(1).setHighlightRequireFieldMatch(true); Integer fragSize = systemConfig.getSearchHighlightFragmentSize(); if (fragSize != null) { solrQuery.setHighlightFragsize(fragSize); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrField.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrField.java index ca9805b6c57..7092a01beb1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrField.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrField.java @@ -63,7 +63,7 @@ public enum SolrType { * support range queries) in * https://github.com/IQSS/dataverse/issues/370 */ - STRING("string"), TEXT_EN("text_en"), INTEGER("int"), LONG("long"), DATE("text_en"), EMAIL("text_en"); + STRING("string"), TEXT_EN("text_en"), INTEGER("plong"), FLOAT("pdouble"), DATE("date_range"), EMAIL("text_en"); private String type; From 3f5919b6ac5eced7b7bfa25eb6ce1a2f3b448326 Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Fri, 27 Sep 2024 15:43:05 +0200 Subject: [PATCH 0052/1048] docs: add release note snippet for #10887 --- doc/release-notes/10887-solr-field-types.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/release-notes/10887-solr-field-types.md diff --git a/doc/release-notes/10887-solr-field-types.md b/doc/release-notes/10887-solr-field-types.md new file mode 100644 index 00000000000..ca5b210cb21 --- /dev/null +++ b/doc/release-notes/10887-solr-field-types.md @@ -0,0 +1,11 @@ +This release enhances how numerical and date fields are indexed in Solr. Previously, all fields were indexed as English text (text_en), but with this update: + +* Integer fields are indexed as `plong` +* Float fields are indexed as `pdouble` +* Date fields are indexed as `date_range` (`solr.DateRangeField`) + +This enables range queries via the search bar or API, such as `exampleIntegerField:[25 TO 50]` or `exampleDateField:[2000-11-01 TO 2014-12-01]`. + +To activate this feature, Dataverse administrators must update their Solr schema.xml (manually or by rerunning `update-fields.sh`) and reindex all datasets. + +Additionally, search result highlighting is now more accurate, ensuring that only fields relevant to the query are highlighted in search results. If the query is specifically limited to certain fields, the highlighting is now limited to those fields as well. \ No newline at end of file From dc8a65e9ff27e83276970247f5b2412ce2d5b3fb Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 27 Sep 2024 09:44:50 -0400 Subject: [PATCH 0053/1048] get image working for cookie consent --- doc/sphinx-guides/source/installation/config.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index bfb444ad6f9..1e585ea4309 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1965,7 +1965,9 @@ To allow users to opt out of the use Google Analytics tracking you can do the fo After restarting or reloading Dataverse the cookie consent popup should appear. -.. |cokkieconsent| image:: ./img/cookie-consent-example.png +|cookieconsent| + +.. |cookieconsent| image:: ./img/cookie-consent-example.png :class: img-responsive If you change the cookie consent config in ``CookieConsent.run()`` and want to test you changes, you should remove the cookie called ``cc_cooke`` in your browser and reload the Dataverse page to have the popup appear again. To remove cookies use Application > Cookies in the Chrome/Edge dev tool, and Storage > Cookies in Firefox and Safari. From 5a3e3d38a90878d81fbcf010a2485818249b665a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 27 Sep 2024 09:59:18 -0400 Subject: [PATCH 0054/1048] tweak cookie consent wording #10320 --- doc/sphinx-guides/source/installation/config.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 1e585ea4309..43358fb4d8a 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1850,12 +1850,12 @@ For Google Analytics, the example script at :download:`analytics-code.html `` -3. Go to https://playground.cookieconsent.orestbida.com/ to configure, download and copy contents of ``cookieconsent-config.js`` to ``analytics-code.html``. It should look something like this: +3. Go to https://playground.cookieconsent.orestbida.com to configure, download and copy contents of ``cookieconsent-config.js`` to ``analytics-code.html``. It should look something like this: .. code-block:: html @@ -1963,7 +1963,7 @@ To allow users to opt out of the use Google Analytics tracking you can do the fo }); -After restarting or reloading Dataverse the cookie consent popup should appear. +After restarting or reloading Dataverse the cookie consent popup should appear, looking something like this: |cookieconsent| From cfd9da8af1c7ad522381eff38c2750e668eb24b8 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 3 Oct 2024 12:55:39 -0400 Subject: [PATCH 0055/1048] added api for optimized lookup for a user --- .../iq/dataverse/PermissionServiceBean.java | 49 ++++++++++++ .../edu/harvard/iq/dataverse/api/Users.java | 24 +++++- .../GetUserPermittedCollectionsCommand.java | 60 ++++++++++++++ .../edu/harvard/iq/dataverse/api/UsersIT.java | 78 +++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 9 +++ 5 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserPermittedCollectionsCommand.java diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index a389cbc735b..cf4aae57174 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -100,6 +100,45 @@ public class PermissionServiceBean { @Inject DatasetVersionFilesServiceBean datasetVersionFilesServiceBean; + private static final String LIST_ALL_DATAVERSES_USER_HAS_PERMISSION = "WITH grouplist AS (\n" + + " SELECT explicitgroup_authenticateduser.explicitgroup_id as id FROM explicitgroup_authenticateduser\n" + + " WHERE explicitgroup_authenticateduser.containedauthenticatedusers_id = @USERID\n" + + ")\n" + + "\n" + + "SELECT * FROM DATAVERSE WHERE id IN (\n" + + " SELECT definitionpoint_id \n" + + " FROM roleassignment\n" + + " WHERE roleassignment.assigneeidentifier IN (\n" + + " SELECT CONCAT('&explicit/', explicitgroup.groupalias) as assignee\n" + + " FROM explicitgroup\n" + + " WHERE explicitgroup.id IN (\n" + + " (\n" + + " SELECT explicitgroup.id id\n" + + " FROM explicitgroup \n" + + " WHERE explicitgroup.id IN (SELECT id FROM grouplist)\n" + + " ) UNION (\n" + + " SELECT explicitgroup_explicitgroup.containedexplicitgroups_id id\n" + + " FROM explicitgroup_explicitgroup\n" + + " WHERE explicitgroup_explicitgroup.explicitgroup_id IN (SELECT id FROM grouplist)\n" + + " AND \n" + + " (SELECT count(*)\n" + + " FROM dataverserole\n" + + " WHERE dataverserole.id = roleassignment.role_id and (dataverserole.permissionbits & @PERMISSIONBIT !=0)) > 0\n" + + " )\n" + + " )\n" + + " ) UNION (\n" + + " SELECT definitionpoint_id \n" + + " FROM roleassignment\n" + + " WHERE roleassignment.assigneeidentifier = (\n" + + " SELECT CONCAT('@', authenticateduser.useridentifier)\n" + + " FROM authenticateduser \n" + + " WHERE authenticateduser.id = @USERID)\n" + + " AND \n" + + " (SELECT count(*)\n" + + " FROM dataverserole\n" + + " WHERE dataverserole.id = roleassignment.role_id and (dataverserole.permissionbits & @PERMISSIONBIT !=0)) > 0\n" + + " )\n" + + ")"; /** * A request-level permission query (e.g includes IP ras). */ @@ -888,4 +927,14 @@ private boolean hasUnrestrictedReleasedFiles(DatasetVersion targetDatasetVersion Long result = em.createQuery(criteriaQuery).getSingleResult(); return result > 0; } + + public List findPermittedCollections(AuthenticatedUser user, int permissionBit) { + if (user != null) { + String sqlCode = LIST_ALL_DATAVERSES_USER_HAS_PERMISSION + .replace("@USERID", String.valueOf(user.getId())) + .replace("@PERMISSIONBIT", String.valueOf(permissionBit)); + return em.createNativeQuery(sqlCode, Dataverse.class).getResultList(); + } + return null; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java index c1a7c95dbff..6a0bf8857a8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java @@ -10,6 +10,7 @@ import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.engine.command.impl.ChangeUserIdentifierCommand; +import edu.harvard.iq.dataverse.engine.command.impl.GetUserPermittedCollectionsCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetUserTracesCommand; import edu.harvard.iq.dataverse.engine.command.impl.MergeInAccountCommand; import edu.harvard.iq.dataverse.engine.command.impl.RevokeAllRolesCommand; @@ -260,5 +261,26 @@ public Response getTracesElement(@Context ContainerRequestContext crc, @Context return ex.getResponse(); } } - + @GET + @AuthRequired + @Path("{identifier}/allowedcollections/{permission}") + @Produces("text/csv, application/json") + public Response getUserPermittedCollections(@Context ContainerRequestContext crc, @Context Request req, @PathParam("identifier") String identifier, @PathParam("permission") String permission) { + AuthenticatedUser authenticatedUser = null; + try { + authenticatedUser = getRequestAuthenticatedUserOrDie(crc); + if (!authenticatedUser.isSuperuser()) { + return error(Response.Status.FORBIDDEN, "This API call can be used by superusers only"); + } + } catch (WrappedResponse ex) { + return error(Response.Status.UNAUTHORIZED, "Authentication is required."); + } + try { + AuthenticatedUser userToQuery = authSvc.getAuthenticatedUser(identifier); + JsonObjectBuilder jsonObj = execCommand(new GetUserPermittedCollectionsCommand(createDataverseRequest(getRequestUser(crc)), userToQuery, permission)); + return ok(jsonObj); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserPermittedCollectionsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserPermittedCollectionsCommand.java new file mode 100644 index 00000000000..8974baf1ced --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserPermittedCollectionsCommand.java @@ -0,0 +1,60 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import jakarta.json.Json; +import jakarta.json.JsonArrayBuilder; +import jakarta.json.JsonObjectBuilder; + +import java.util.List; +import java.util.logging.Logger; + +import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; + +@RequiredPermissions({}) +public class GetUserPermittedCollectionsCommand extends AbstractCommand { + private static final Logger logger = Logger.getLogger(GetUserPermittedCollectionsCommand.class.getCanonicalName()); + + private DataverseRequest request; + private AuthenticatedUser user; + private String permission; + public GetUserPermittedCollectionsCommand(DataverseRequest request, AuthenticatedUser user, String permission) { + super(request, (DvObject) null); + this.request = request; + this.user = user; + this.permission = permission; + } + + @Override + public JsonObjectBuilder execute(CommandContext ctxt) throws CommandException { + if (user == null) { + throw new CommandException("User not found.", this); + } + int permissionBit; + try { + permissionBit = permission.equalsIgnoreCase("any") ? + Integer.MAX_VALUE : (1 << Permission.valueOf(permission).ordinal()); + } catch (IllegalArgumentException e) { + throw new CommandException("Permission not valid.", this); + } + List collections = ctxt.permissions().findPermittedCollections(user, permissionBit); + if (collections != null) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (Dataverse dv : collections) { + jab.add(json(dv)); + } + job.add("count", collections.size()); + job.add("items", jab); + return job; + } + return null; + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java index 1003c1a990c..438dc74461b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java @@ -1,5 +1,7 @@ package edu.harvard.iq.dataverse.api; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.util.BundleUtil; import io.restassured.RestAssured; import static io.restassured.RestAssured.given; import io.restassured.http.ContentType; @@ -516,6 +518,82 @@ public void testDeleteAuthenticatedUser() { } + @Test + public void testUserPermittedDataverses() { + Response createSuperuser = UtilIT.createRandomUser(); + String superuserUsername = UtilIT.getUsernameFromResponse(createSuperuser); + String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); + Response toggleSuperuser = UtilIT.makeSuperUser(superuserUsername); + toggleSuperuser.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + assertEquals(200, createUser.getStatusCode()); + String usernameOfUser = UtilIT.getUsernameFromResponse(createUser); + String userApiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverse1 = UtilIT.createRandomDataverse(superuserApiToken); + createDataverse1.prettyPrint(); + createDataverse1.then().assertThat() + .statusCode(CREATED.getStatusCode()); + String dataverseAlias1 = UtilIT.getAliasFromResponse(createDataverse1); + + // create a second Dataverse and add a Group with permissions + Response createDataverse2 = UtilIT.createRandomDataverse(superuserApiToken); + createDataverse2.prettyPrint(); + createDataverse2.then().assertThat() + .statusCode(CREATED.getStatusCode()); + String dataverseAlias2 = UtilIT.getAliasFromResponse(createDataverse2); + String aliasInOwner = "groupFor" + dataverseAlias2; + String displayName = "Group for " + dataverseAlias2; + Response createGroup = UtilIT.createGroup(dataverseAlias2, aliasInOwner, displayName, superuserApiToken); + String groupIdentifier = JsonPath.from(createGroup.asString()).getString("data.identifier"); + Response grantRoleResponse = UtilIT.grantRoleOnDataverse(dataverseAlias2, DataverseRole.EDITOR.toString(), groupIdentifier, superuserApiToken); + grantRoleResponse.prettyPrint(); + grantRoleResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response collectionsResp = UtilIT.getUserPermittedCollections(usernameOfUser, userApiToken, "ViewUnpublishedDataset"); + assertEquals(403, collectionsResp.getStatusCode()); + collectionsResp = UtilIT.getUserPermittedCollections(usernameOfUser, "", "ViewUnpublishedDataset"); + assertEquals(401, collectionsResp.getStatusCode()); + collectionsResp = UtilIT.getUserPermittedCollections("fakeUser", superuserApiToken, "ViewUnpublishedDataset"); + assertEquals(500, collectionsResp.getStatusCode()); + collectionsResp = UtilIT.getUserPermittedCollections(usernameOfUser, superuserApiToken, "bad"); + assertEquals(500, collectionsResp.getStatusCode()); + + // Testing adding an explicit permission/role to one dataverse + collectionsResp = UtilIT.getUserPermittedCollections(usernameOfUser, superuserApiToken, "DownloadFile"); + collectionsResp.prettyPrint(); + collectionsResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.count", equalTo(0)); + + Response assignRole = UtilIT.grantRoleOnDataverse(dataverseAlias1, DataverseRole.EDITOR.toString(), + "@" + usernameOfUser, superuserApiToken); + assignRole.prettyPrint(); + assertEquals(200, assignRole.getStatusCode()); + + collectionsResp = UtilIT.getUserPermittedCollections(usernameOfUser, superuserApiToken, "DownloadFile"); + collectionsResp.prettyPrint(); + collectionsResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.count", equalTo(1)); + + // Add user to group and test with both explicit and group permissions + Response addToGroup = UtilIT.addToGroup(dataverseAlias2, aliasInOwner, List.of("@" + usernameOfUser), superuserApiToken); + addToGroup.prettyPrint(); + addToGroup.then().assertThat() + .statusCode(OK.getStatusCode()); + + collectionsResp = UtilIT.getUserPermittedCollections(usernameOfUser, superuserApiToken, "DownloadFile"); + collectionsResp.prettyPrint(); + collectionsResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.count", equalTo(2)); + } + private Response convertUserFromBcryptToSha1(long idOfBcryptUserToConvert, String password) { JsonObjectBuilder data = Json.createObjectBuilder(); data.add("builtinUserId", idOfBcryptUserToConvert); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 70f49d81b35..302bd751d45 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1344,6 +1344,15 @@ public static Response getUserTraces(String username, String apiToken) { return response; } + public static Response getUserPermittedCollections(String username, String apiToken, String permission) { + RequestSpecification requestSpecification = given(); + if (!StringUtil.isEmpty(apiToken)) { + requestSpecification.header(API_TOKEN_HTTP_HEADER, apiToken); + } + Response response = requestSpecification.get("/api/users/" + username + "/allowedcollections/" + permission); + return response; + } + public static Response reingestFile(Long fileId, String apiToken) { Response response = given() .header(API_TOKEN_HTTP_HEADER, apiToken) From 7f8ed30bf39792045dfc667a9f489586eac6ed31 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 3 Oct 2024 13:23:07 -0400 Subject: [PATCH 0056/1048] add release note --- .../6467-optimize-permission-lookups-for-a-user.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/release-notes/6467-optimize-permission-lookups-for-a-user.md diff --git a/doc/release-notes/6467-optimize-permission-lookups-for-a-user.md b/doc/release-notes/6467-optimize-permission-lookups-for-a-user.md new file mode 100644 index 00000000000..593af3a6d35 --- /dev/null +++ b/doc/release-notes/6467-optimize-permission-lookups-for-a-user.md @@ -0,0 +1,9 @@ +The following API have been added: + +/api/users/{identifier}/allowedcollections/{permission} + +This API lists the dataverses/collections that the user has access to via the permission passed. +By passing "any" as the permission the list will return all dataverse/collections that the user can access regardless of which permission is used. +This API can be executed only by administrators. +Valid Permissions are: AddDataverse, AddDataset, ViewUnpublishedDataverse, ViewUnpublishedDataset, DownloadFile, EditDataverse, EditDataset, ManageDataversePermissions, +ManageDatasetPermissions, ManageFilePermissions, PublishDataverse, PublishDataset, DeleteDataverse, DeleteDatasetDraft, and "any" as a wildcard option. From 66c112d889b9805857f72751b5780860814b5e21 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 3 Oct 2024 13:39:12 -0400 Subject: [PATCH 0057/1048] removed unused include --- src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java index 438dc74461b..6ac0957ae8e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java @@ -1,7 +1,5 @@ package edu.harvard.iq.dataverse.api; -import edu.harvard.iq.dataverse.authorization.Permission; -import edu.harvard.iq.dataverse.util.BundleUtil; import io.restassured.RestAssured; import static io.restassured.RestAssured.given; import io.restassured.http.ContentType; From e51cf6ca8176510807efb992561f2b8b957f0eae Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 3 Oct 2024 15:47:38 -0400 Subject: [PATCH 0058/1048] optimize sql --- .../iq/dataverse/PermissionServiceBean.java | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index cf4aae57174..9947cc33775 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -106,7 +106,7 @@ public class PermissionServiceBean { ")\n" + "\n" + "SELECT * FROM DATAVERSE WHERE id IN (\n" + - " SELECT definitionpoint_id \n" + + " SELECT definitionpoint_id\n" + " FROM roleassignment\n" + " WHERE roleassignment.assigneeidentifier IN (\n" + " SELECT CONCAT('&explicit/', explicitgroup.groupalias) as assignee\n" + @@ -114,29 +114,25 @@ public class PermissionServiceBean { " WHERE explicitgroup.id IN (\n" + " (\n" + " SELECT explicitgroup.id id\n" + - " FROM explicitgroup \n" + - " WHERE explicitgroup.id IN (SELECT id FROM grouplist)\n" + + " FROM explicitgroup\n" + + " WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup.id)\n" + " ) UNION (\n" + " SELECT explicitgroup_explicitgroup.containedexplicitgroups_id id\n" + " FROM explicitgroup_explicitgroup\n" + - " WHERE explicitgroup_explicitgroup.explicitgroup_id IN (SELECT id FROM grouplist)\n" + - " AND \n" + - " (SELECT count(*)\n" + - " FROM dataverserole\n" + - " WHERE dataverserole.id = roleassignment.role_id and (dataverserole.permissionbits & @PERMISSIONBIT !=0)) > 0\n" + + " WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup_explicitgroup.explicitgroup_id)\n" + + " AND EXISTS (SELECT id FROM dataverserole\n" + + " WHERE dataverserole.id = roleassignment.role_id and (dataverserole.permissionbits & @PERMISSIONBIT !=0))\n" + " )\n" + " )\n" + " ) UNION (\n" + - " SELECT definitionpoint_id \n" + + " SELECT definitionpoint_id\n" + " FROM roleassignment\n" + " WHERE roleassignment.assigneeidentifier = (\n" + " SELECT CONCAT('@', authenticateduser.useridentifier)\n" + - " FROM authenticateduser \n" + + " FROM authenticateduser\n" + " WHERE authenticateduser.id = @USERID)\n" + - " AND \n" + - " (SELECT count(*)\n" + - " FROM dataverserole\n" + - " WHERE dataverserole.id = roleassignment.role_id and (dataverserole.permissionbits & @PERMISSIONBIT !=0)) > 0\n" + + " AND EXISTS (SELECT id FROM dataverserole\n" + + " WHERE dataverserole.id = roleassignment.role_id and (dataverserole.permissionbits & @PERMISSIONBIT !=0))\n" + " )\n" + ")"; /** From 3b860e5c6cb0628f9f6f1a5269e33054d5a9297e Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 4 Oct 2024 11:43:25 -0400 Subject: [PATCH 0059/1048] Update doc/release-notes/6467-optimize-permission-lookups-for-a-user.md Co-authored-by: Philip Durbin --- .../6467-optimize-permission-lookups-for-a-user.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/6467-optimize-permission-lookups-for-a-user.md b/doc/release-notes/6467-optimize-permission-lookups-for-a-user.md index 593af3a6d35..7cc3f56d762 100644 --- a/doc/release-notes/6467-optimize-permission-lookups-for-a-user.md +++ b/doc/release-notes/6467-optimize-permission-lookups-for-a-user.md @@ -4,6 +4,6 @@ The following API have been added: This API lists the dataverses/collections that the user has access to via the permission passed. By passing "any" as the permission the list will return all dataverse/collections that the user can access regardless of which permission is used. -This API can be executed only by administrators. +This API can be executed only by superusers. Valid Permissions are: AddDataverse, AddDataset, ViewUnpublishedDataverse, ViewUnpublishedDataset, DownloadFile, EditDataverse, EditDataset, ManageDataversePermissions, ManageDatasetPermissions, ManageFilePermissions, PublishDataverse, PublishDataset, DeleteDataverse, DeleteDatasetDraft, and "any" as a wildcard option. From 83c01d13eb5a4d05b05dc6fae1e98497bb21e8e1 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 4 Oct 2024 12:03:13 -0400 Subject: [PATCH 0060/1048] reformattting per review comments --- .../iq/dataverse/PermissionServiceBean.java | 72 ++++++++++--------- .../edu/harvard/iq/dataverse/api/Users.java | 4 +- .../edu/harvard/iq/dataverse/api/UtilIT.java | 2 +- 3 files changed, 40 insertions(+), 38 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index 9947cc33775..31ae0966a8e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -100,41 +100,43 @@ public class PermissionServiceBean { @Inject DatasetVersionFilesServiceBean datasetVersionFilesServiceBean; - private static final String LIST_ALL_DATAVERSES_USER_HAS_PERMISSION = "WITH grouplist AS (\n" + - " SELECT explicitgroup_authenticateduser.explicitgroup_id as id FROM explicitgroup_authenticateduser\n" + - " WHERE explicitgroup_authenticateduser.containedauthenticatedusers_id = @USERID\n" + - ")\n" + - "\n" + - "SELECT * FROM DATAVERSE WHERE id IN (\n" + - " SELECT definitionpoint_id\n" + - " FROM roleassignment\n" + - " WHERE roleassignment.assigneeidentifier IN (\n" + - " SELECT CONCAT('&explicit/', explicitgroup.groupalias) as assignee\n" + - " FROM explicitgroup\n" + - " WHERE explicitgroup.id IN (\n" + - " (\n" + - " SELECT explicitgroup.id id\n" + - " FROM explicitgroup\n" + - " WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup.id)\n" + - " ) UNION (\n" + - " SELECT explicitgroup_explicitgroup.containedexplicitgroups_id id\n" + - " FROM explicitgroup_explicitgroup\n" + - " WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup_explicitgroup.explicitgroup_id)\n" + - " AND EXISTS (SELECT id FROM dataverserole\n" + - " WHERE dataverserole.id = roleassignment.role_id and (dataverserole.permissionbits & @PERMISSIONBIT !=0))\n" + - " )\n" + - " )\n" + - " ) UNION (\n" + - " SELECT definitionpoint_id\n" + - " FROM roleassignment\n" + - " WHERE roleassignment.assigneeidentifier = (\n" + - " SELECT CONCAT('@', authenticateduser.useridentifier)\n" + - " FROM authenticateduser\n" + - " WHERE authenticateduser.id = @USERID)\n" + - " AND EXISTS (SELECT id FROM dataverserole\n" + - " WHERE dataverserole.id = roleassignment.role_id and (dataverserole.permissionbits & @PERMISSIONBIT !=0))\n" + - " )\n" + - ")"; + private static final String LIST_ALL_DATAVERSES_USER_HAS_PERMISSION = """ + WITH grouplist AS ( + SELECT explicitgroup_authenticateduser.explicitgroup_id as id FROM explicitgroup_authenticateduser + WHERE explicitgroup_authenticateduser.containedauthenticatedusers_id = @USERID + ) + + SELECT * FROM DATAVERSE WHERE id IN ( + SELECT definitionpoint_id\s + FROM roleassignment + WHERE roleassignment.assigneeidentifier IN ( + SELECT CONCAT('&explicit/', explicitgroup.groupalias) as assignee + FROM explicitgroup + WHERE explicitgroup.id IN ( + ( + SELECT explicitgroup.id id + FROM explicitgroup\s + WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup.id) + ) UNION ( + SELECT explicitgroup_explicitgroup.containedexplicitgroups_id id + FROM explicitgroup_explicitgroup + WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup_explicitgroup.explicitgroup_id) + AND EXISTS (SELECT id FROM dataverserole + WHERE dataverserole.id = roleassignment.role_id and (dataverserole.permissionbits & @PERMISSIONBIT !=0)) + ) + ) + ) UNION ( + SELECT definitionpoint_id\s + FROM roleassignment + WHERE roleassignment.assigneeidentifier = ( + SELECT CONCAT('@', authenticateduser.useridentifier) + FROM authenticateduser\s + WHERE authenticateduser.id = @USERID) + AND EXISTS (SELECT id FROM dataverserole + WHERE dataverserole.id = roleassignment.role_id and (dataverserole.permissionbits & @PERMISSIONBIT !=0)) + ) + ) + """; /** * A request-level permission query (e.g includes IP ras). */ diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java index 6a0bf8857a8..01c92941224 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java @@ -263,8 +263,8 @@ public Response getTracesElement(@Context ContainerRequestContext crc, @Context } @GET @AuthRequired - @Path("{identifier}/allowedcollections/{permission}") - @Produces("text/csv, application/json") + @Path("{identifier}/allowedCollections/{permission}") + @Produces("application/json") public Response getUserPermittedCollections(@Context ContainerRequestContext crc, @Context Request req, @PathParam("identifier") String identifier, @PathParam("permission") String permission) { AuthenticatedUser authenticatedUser = null; try { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 302bd751d45..ed420b53dd3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1349,7 +1349,7 @@ public static Response getUserPermittedCollections(String username, String apiTo if (!StringUtil.isEmpty(apiToken)) { requestSpecification.header(API_TOKEN_HTTP_HEADER, apiToken); } - Response response = requestSpecification.get("/api/users/" + username + "/allowedcollections/" + permission); + Response response = requestSpecification.get("/api/users/" + username + "/allowedCollections/" + permission); return response; } From 728835eb1c120b44ed4cfc8c95c5910f9c507a55 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 4 Oct 2024 13:49:40 -0400 Subject: [PATCH 0061/1048] remove tabs from sql --- doc/sphinx-guides/source/api/native-api.rst | 21 +++++++ .../iq/dataverse/PermissionServiceBean.java | 60 +++++++++---------- 2 files changed, 51 insertions(+), 30 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index f8b8620f121..da9823d81c2 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -6012,6 +6012,27 @@ Example: List permissions a user (based on API Token used) has on a dataset whos curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/admin/permissions/:persistentId?persistentId=$PERSISTENT_IDENTIFIER" +List Dataverse collections a user can act on based on their permissions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +List Dataverse collections a user can act on based on a particular permission :: + + GET http://$SERVER/api/users/$identifier/allowedCollections/$permission + +.. note:: This API can only be called by an Administrator + +The ``$identifier`` is the username of the requested user. +The ``$permission`` is the permission (tied to the roles) that gives the user access to the collection. +Passing ``$permission`` as 'any' will return the collection as long as the user has any access/permission on the collection + +.. code-block:: bash + + export SERVER_URL=https://demo.dataverse.org + export $USERNAME=jsmith + export PERMISSION=PublishDataverse + + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/users/$USERNAME/allowedCollections/$PERMISSION" + Show Role Assignee ~~~~~~~~~~~~~~~~~~ diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index 31ae0966a8e..7cc1dc5b9ca 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -102,39 +102,39 @@ public class PermissionServiceBean { private static final String LIST_ALL_DATAVERSES_USER_HAS_PERMISSION = """ WITH grouplist AS ( - SELECT explicitgroup_authenticateduser.explicitgroup_id as id FROM explicitgroup_authenticateduser - WHERE explicitgroup_authenticateduser.containedauthenticatedusers_id = @USERID + SELECT explicitgroup_authenticateduser.explicitgroup_id as id FROM explicitgroup_authenticateduser + WHERE explicitgroup_authenticateduser.containedauthenticatedusers_id = @USERID ) SELECT * FROM DATAVERSE WHERE id IN ( - SELECT definitionpoint_id\s - FROM roleassignment - WHERE roleassignment.assigneeidentifier IN ( - SELECT CONCAT('&explicit/', explicitgroup.groupalias) as assignee - FROM explicitgroup - WHERE explicitgroup.id IN ( - ( - SELECT explicitgroup.id id - FROM explicitgroup\s - WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup.id) - ) UNION ( - SELECT explicitgroup_explicitgroup.containedexplicitgroups_id id - FROM explicitgroup_explicitgroup - WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup_explicitgroup.explicitgroup_id) - AND EXISTS (SELECT id FROM dataverserole - WHERE dataverserole.id = roleassignment.role_id and (dataverserole.permissionbits & @PERMISSIONBIT !=0)) - ) - ) - ) UNION ( - SELECT definitionpoint_id\s - FROM roleassignment - WHERE roleassignment.assigneeidentifier = ( - SELECT CONCAT('@', authenticateduser.useridentifier) - FROM authenticateduser\s - WHERE authenticateduser.id = @USERID) - AND EXISTS (SELECT id FROM dataverserole - WHERE dataverserole.id = roleassignment.role_id and (dataverserole.permissionbits & @PERMISSIONBIT !=0)) - ) + SELECT definitionpoint_id\s + FROM roleassignment + WHERE roleassignment.assigneeidentifier IN ( + SELECT CONCAT('&explicit/', explicitgroup.groupalias) as assignee + FROM explicitgroup + WHERE explicitgroup.id IN ( + ( + SELECT explicitgroup.id id + FROM explicitgroup\s + WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup.id) + ) UNION ( + SELECT explicitgroup_explicitgroup.containedexplicitgroups_id id + FROM explicitgroup_explicitgroup + WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup_explicitgroup.explicitgroup_id) + AND EXISTS (SELECT id FROM dataverserole + WHERE dataverserole.id = roleassignment.role_id and (dataverserole.permissionbits & @PERMISSIONBIT !=0)) + ) + ) + ) UNION ( + SELECT definitionpoint_id\s + FROM roleassignment + WHERE roleassignment.assigneeidentifier = ( + SELECT CONCAT('@', authenticateduser.useridentifier) + FROM authenticateduser\s + WHERE authenticateduser.id = @USERID) + AND EXISTS (SELECT id FROM dataverserole + WHERE dataverserole.id = roleassignment.role_id and (dataverserole.permissionbits & @PERMISSIONBIT !=0)) + ) ) """; /** From 7257e7d24958693c02d97bfdb418f246799f7b81 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 4 Oct 2024 15:56:57 -0400 Subject: [PATCH 0062/1048] adding :authenticated-users to SQl and swapping out ServiceDocumentManagerImpl call for new call --- .../iq/dataverse/PermissionServiceBean.java | 101 +++++++----------- .../ServiceDocumentManagerImpl.java | 2 +- 2 files changed, 41 insertions(+), 62 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index 7cc1dc5b9ca..0e7bf73219d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -102,40 +102,46 @@ public class PermissionServiceBean { private static final String LIST_ALL_DATAVERSES_USER_HAS_PERMISSION = """ WITH grouplist AS ( - SELECT explicitgroup_authenticateduser.explicitgroup_id as id FROM explicitgroup_authenticateduser - WHERE explicitgroup_authenticateduser.containedauthenticatedusers_id = @USERID - ) + SELECT explicitgroup_authenticateduser.explicitgroup_id as id FROM explicitgroup_authenticateduser + WHERE explicitgroup_authenticateduser.containedauthenticatedusers_id = 6 + ) - SELECT * FROM DATAVERSE WHERE id IN ( - SELECT definitionpoint_id\s - FROM roleassignment - WHERE roleassignment.assigneeidentifier IN ( - SELECT CONCAT('&explicit/', explicitgroup.groupalias) as assignee - FROM explicitgroup - WHERE explicitgroup.id IN ( - ( - SELECT explicitgroup.id id - FROM explicitgroup\s - WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup.id) + SELECT * FROM DATAVERSE WHERE id IN ( + SELECT definitionpoint_id + FROM roleassignment + WHERE roleassignment.assigneeidentifier IN ( + SELECT CONCAT('&explicit/', explicitgroup.groupalias) as assignee + FROM explicitgroup + WHERE explicitgroup.id IN ( + ( + SELECT explicitgroup.id id + FROM explicitgroup + WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup.id) + ) UNION ( + SELECT explicitgroup_explicitgroup.containedexplicitgroups_id id + FROM explicitgroup_explicitgroup + WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup_explicitgroup.explicitgroup_id) + AND EXISTS (SELECT id FROM dataverserole + WHERE dataverserole.id = roleassignment.role_id AND (dataverserole.permissionbits & 2 !=0)) + ) + ) ) UNION ( - SELECT explicitgroup_explicitgroup.containedexplicitgroups_id id - FROM explicitgroup_explicitgroup - WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup_explicitgroup.explicitgroup_id) - AND EXISTS (SELECT id FROM dataverserole - WHERE dataverserole.id = roleassignment.role_id and (dataverserole.permissionbits & @PERMISSIONBIT !=0)) - ) + SELECT definitionpoint_id + FROM roleassignment + WHERE roleassignment.assigneeidentifier = ( + SELECT CONCAT('@', authenticateduser.useridentifier) + FROM authenticateduser + WHERE authenticateduser.id = 6) + AND EXISTS (SELECT id FROM dataverserole + WHERE dataverserole.id = roleassignment.role_id AND (dataverserole.permissionbits & 2 !=0)) + ) UNION ( + SELECT definitionpoint_id + FROM roleassignment + WHERE roleassignment.assigneeidentifier = ':authenticated-users' + AND EXISTS (SELECT id FROM dataverserole + WHERE dataverserole.id = roleassignment.role_id AND (dataverserole.permissionbits & 2 !=0)) + ) ) - ) UNION ( - SELECT definitionpoint_id\s - FROM roleassignment - WHERE roleassignment.assigneeidentifier = ( - SELECT CONCAT('@', authenticateduser.useridentifier) - FROM authenticateduser\s - WHERE authenticateduser.id = @USERID) - AND EXISTS (SELECT id FROM dataverserole - WHERE dataverserole.id = roleassignment.role_id and (dataverserole.permissionbits & @PERMISSIONBIT !=0)) - ) - ) """; /** * A request-level permission query (e.g includes IP ras). @@ -590,36 +596,6 @@ public RequestPermissionQuery request(DataverseRequest req) { return new RequestPermissionQuery(null, req); } - /** - * Go from (User, Permission) to a list of Dataverse objects that the user - has the permission on. - * - * @param user - * @param permission - * @return The list of dataverses {@code user} has permission - {@code permission} on. - */ - public List getDataversesUserHasPermissionOn(AuthenticatedUser user, Permission permission) { - Set groups = groupService.groupsFor(user); - String identifiers = GroupUtil.getAllIdentifiersForUser(user, groups); - /** - * @todo Are there any strings in identifiers that would break this SQL - * query? - */ - String query = "SELECT id FROM dvobject WHERE dtype = 'Dataverse' and id in (select definitionpoint_id from roleassignment where assigneeidentifier in (" + identifiers + "));"; - logger.log(Level.FINE, "query: {0}", query); - Query nativeQuery = em.createNativeQuery(query); - List dataverseIdsToCheck = nativeQuery.getResultList(); - List dataversesUserHasPermissionOn = new LinkedList<>(); - for (int dvIdAsInt : dataverseIdsToCheck) { - Dataverse dataverse = dataverseService.find(Long.valueOf(dvIdAsInt)); - if (userOn(user, dataverse).has(permission)) { - dataversesUserHasPermissionOn.add(dataverse); - } - } - return dataversesUserHasPermissionOn; - } - public List getUsersWithPermissionOn(Permission permission, DvObject dvo) { List usersHasPermissionOn = new LinkedList<>(); Set ras = roleService.rolesAssignments(dvo); @@ -926,6 +902,9 @@ private boolean hasUnrestrictedReleasedFiles(DatasetVersion targetDatasetVersion return result > 0; } + public List findPermittedCollections(AuthenticatedUser user, Permission permission) { + return findPermittedCollections(user, 1 << permission.ordinal()); + } public List findPermittedCollections(AuthenticatedUser user, int permissionBit) { if (user != null) { String sqlCode = LIST_ALL_DATAVERSES_USER_HAS_PERMISSION diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java index 134d54aef88..4badf3cbdb4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java @@ -65,7 +65,7 @@ public ServiceDocument getServiceDocument(String sdUri, AuthCredentials authCred * shibIdentityProvider String on AuthenticatedUser is only set when a * SAML assertion is made at runtime via the browser. */ - List dataverses = permissionService.getDataversesUserHasPermissionOn(user, Permission.AddDataset); + List dataverses = permissionService.findPermittedCollections(user, Permission.AddDataset); for (Dataverse dataverse : dataverses) { String dvAlias = dataverse.getAlias(); if (dvAlias != null && !dvAlias.isEmpty()) { From a194138743dea370250c076fb581cb7e2ef3c08c Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 7 Oct 2024 10:03:01 -0400 Subject: [PATCH 0063/1048] fix typo --- .../harvard/iq/dataverse/PermissionServiceBean.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index 0e7bf73219d..629e1336d0f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -103,7 +103,7 @@ public class PermissionServiceBean { private static final String LIST_ALL_DATAVERSES_USER_HAS_PERMISSION = """ WITH grouplist AS ( SELECT explicitgroup_authenticateduser.explicitgroup_id as id FROM explicitgroup_authenticateduser - WHERE explicitgroup_authenticateduser.containedauthenticatedusers_id = 6 + WHERE explicitgroup_authenticateduser.containedauthenticatedusers_id = @USERID ) SELECT * FROM DATAVERSE WHERE id IN ( @@ -122,7 +122,7 @@ WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup.id) FROM explicitgroup_explicitgroup WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup_explicitgroup.explicitgroup_id) AND EXISTS (SELECT id FROM dataverserole - WHERE dataverserole.id = roleassignment.role_id AND (dataverserole.permissionbits & 2 !=0)) + WHERE dataverserole.id = roleassignment.role_id AND (dataverserole.permissionbits & @PERMISSIONBIT !=0)) ) ) ) UNION ( @@ -131,15 +131,15 @@ AND EXISTS (SELECT id FROM dataverserole WHERE roleassignment.assigneeidentifier = ( SELECT CONCAT('@', authenticateduser.useridentifier) FROM authenticateduser - WHERE authenticateduser.id = 6) + WHERE authenticateduser.id = @USERID) AND EXISTS (SELECT id FROM dataverserole - WHERE dataverserole.id = roleassignment.role_id AND (dataverserole.permissionbits & 2 !=0)) + WHERE dataverserole.id = roleassignment.role_id AND (dataverserole.permissionbits & @PERMISSIONBIT !=0)) ) UNION ( SELECT definitionpoint_id FROM roleassignment WHERE roleassignment.assigneeidentifier = ':authenticated-users' AND EXISTS (SELECT id FROM dataverserole - WHERE dataverserole.id = roleassignment.role_id AND (dataverserole.permissionbits & 2 !=0)) + WHERE dataverserole.id = roleassignment.role_id AND (dataverserole.permissionbits & @PERMISSIONBIT !=0)) ) ) """; From 423077d48758f217e7fadbf79d5591772841a424 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:34:51 -0400 Subject: [PATCH 0064/1048] adding IP Group permission access --- .../iq/dataverse/PermissionServiceBean.java | 54 +++++++++++++++---- .../edu/harvard/iq/dataverse/api/Users.java | 4 +- .../ServiceDocumentManagerImpl.java | 5 +- .../GetUserPermittedCollectionsCommand.java | 3 +- .../harvard/iq/dataverse/api/IpGroupsIT.java | 16 ++++-- .../edu/harvard/iq/dataverse/api/UsersIT.java | 12 +++-- 6 files changed, 73 insertions(+), 21 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index 629e1336d0f..ee5e2aa9b55 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -2,13 +2,14 @@ import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.DataverseRole; +import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IPv4Address; +import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IPv6Address; +import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; import edu.harvard.iq.dataverse.authorization.users.GuestUser; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.RoleAssignee; -import edu.harvard.iq.dataverse.authorization.groups.Group; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; -import edu.harvard.iq.dataverse.authorization.groups.GroupUtil; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.engine.command.Command; @@ -37,7 +38,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; -import java.util.logging.Level; import java.util.stream.Collectors; import static java.util.stream.Collectors.toList; import jakarta.persistence.Query; @@ -134,13 +134,24 @@ SELECT CONCAT('@', authenticateduser.useridentifier) WHERE authenticateduser.id = @USERID) AND EXISTS (SELECT id FROM dataverserole WHERE dataverserole.id = roleassignment.role_id AND (dataverserole.permissionbits & @PERMISSIONBIT !=0)) - ) UNION ( + ) UNION ( SELECT definitionpoint_id FROM roleassignment WHERE roleassignment.assigneeidentifier = ':authenticated-users' AND EXISTS (SELECT id FROM dataverserole WHERE dataverserole.id = roleassignment.role_id AND (dataverserole.permissionbits & @PERMISSIONBIT !=0)) - ) + ) UNION ( + SELECT definitionpoint_id + FROM roleassignment + WHERE roleassignment.assigneeidentifier IN ( + SELECT CONCAT('&ip/', persistedglobalgroup.persistedgroupalias) as assignee + FROM persistedglobalgroup + LEFT OUTER JOIN ipv4range ON persistedglobalgroup.id = ipv4range.owner_id + LEFT OUTER JOIN ipv6range ON persistedglobalgroup.id = ipv6range.owner_id + WHERE dtype = 'IpGroup' + AND @IPRANGESQL + ) + ) ) """; /** @@ -902,16 +913,41 @@ private boolean hasUnrestrictedReleasedFiles(DatasetVersion targetDatasetVersion return result > 0; } - public List findPermittedCollections(AuthenticatedUser user, Permission permission) { - return findPermittedCollections(user, 1 << permission.ordinal()); + public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, Permission permission) { + return findPermittedCollections(request, user, 1 << permission.ordinal()); } - public List findPermittedCollections(AuthenticatedUser user, int permissionBit) { + public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit) { if (user != null) { + + // IP Group + IpAddress ip = request != null ? request.getSourceAddress() : new IPv4Address(0L); + String ipRangeSQL = "FALSE"; + if (ip instanceof IPv4Address) { + IPv4Address ipv4 = (IPv4Address) ip; + ipRangeSQL = ipv4.toBigInteger() + " BETWEEN ipv4range.bottomaslong AND ipv4range.topaslong"; + } else if (ip instanceof IPv6Address) { + IPv6Address ipv6 = (IPv6Address) ip; + long[] vals = ipv6.toLongArray(); + if (vals.length == 4) { + ipRangeSQL = """ + (@0 BETWEEN ipv6range.bottoma AND ipv6range.topa + AND @1 BETWEEN ipv6range.bottomb AND ipv6range.topb + AND @2 BETWEEN ipv6range.bottomc AND ipv6range.topc + AND @3 BETWEEN ipv6range.bottomd AND ipv6range.topd) + """; + for (int i = 0; i < vals.length; i++) { + ipRangeSQL = ipRangeSQL.replace("@" + i, String.valueOf(vals[i])); + } + } + } + String sqlCode = LIST_ALL_DATAVERSES_USER_HAS_PERMISSION .replace("@USERID", String.valueOf(user.getId())) - .replace("@PERMISSIONBIT", String.valueOf(permissionBit)); + .replace("@PERMISSIONBIT", String.valueOf(permissionBit)) + .replace("@IPRANGESQL", ipRangeSQL); return em.createNativeQuery(sqlCode, Dataverse.class).getResultList(); } return null; } } + diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java index 01c92941224..3eef7ab0610 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java @@ -269,8 +269,8 @@ public Response getUserPermittedCollections(@Context ContainerRequestContext crc AuthenticatedUser authenticatedUser = null; try { authenticatedUser = getRequestAuthenticatedUserOrDie(crc); - if (!authenticatedUser.isSuperuser()) { - return error(Response.Status.FORBIDDEN, "This API call can be used by superusers only"); + if (!authenticatedUser.getUserIdentifier().equalsIgnoreCase(identifier) && !authenticatedUser.isSuperuser()) { + return error(Response.Status.FORBIDDEN, "This API call can be used by Users getting there own permitted collections or by superusers."); } } catch (WrappedResponse ex) { return error(Response.Status.UNAUTHORIZED, "Authentication is required."); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java index 4badf3cbdb4..423c631aade 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java @@ -5,6 +5,7 @@ import edu.harvard.iq.dataverse.PermissionServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.util.SystemConfig; import java.util.List; import java.util.logging.Logger; @@ -64,8 +65,10 @@ public ServiceDocument getServiceDocument(String sdUri, AuthCredentials authCred * a Shibboleth user can have an API token the transient * shibIdentityProvider String on AuthenticatedUser is only set when a * SAML assertion is made at runtime via the browser. + * + * We also don't support IP Groups since we don't have access to the request containing the ip address. */ - List dataverses = permissionService.findPermittedCollections(user, Permission.AddDataset); + List dataverses = permissionService.findPermittedCollections(null, user, Permission.AddDataset); for (Dataverse dataverse : dataverses) { String dvAlias = dataverse.getAlias(); if (dvAlias != null && !dvAlias.isEmpty()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserPermittedCollectionsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserPermittedCollectionsCommand.java index 8974baf1ced..c4888c8c99c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserPermittedCollectionsCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserPermittedCollectionsCommand.java @@ -3,6 +3,7 @@ import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; @@ -44,7 +45,7 @@ public JsonObjectBuilder execute(CommandContext ctxt) throws CommandException { } catch (IllegalArgumentException e) { throw new CommandException("Permission not valid.", this); } - List collections = ctxt.permissions().findPermittedCollections(user, permissionBit); + List collections = ctxt.permissions().findPermittedCollections(request, user, permissionBit); if (collections != null) { JsonObjectBuilder job = Json.createObjectBuilder(); JsonArrayBuilder jab = Json.createArrayBuilder(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/IpGroupsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/IpGroupsIT.java index 1c7e7b05650..5d54909ac42 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/IpGroupsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/IpGroupsIT.java @@ -18,7 +18,11 @@ import org.junit.jupiter.api.Test; public class IpGroupsIT { - +/* + WARNING: Running this test will creat IP Groups that give access based on any IP address. + This will cause other tests that count the number of Collections a user can access to be higher than expected. + Since this Test Class is not being run in an automated test suite it isn't an issue. + */ private static final Logger logger = Logger.getLogger(IpGroupsIT.class.getCanonicalName()); @BeforeAll @@ -48,12 +52,12 @@ public void testDownloadFile() { Response userWithNoRoles = UtilIT.createRandomUser(); String userWithNoRolesApiToken = UtilIT.getApiTokenFromResponse(userWithNoRoles); - String pathToFile = "src/main/webapp/resources/images/favicondataverse.png"; + String pathToFile = "src/main/webapp/resources/images/dataverseproject.png"; Response addResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); addResponse.then().assertThat() .body("data.files[0].dataFile.contentType", equalTo("image/png")) - .body("data.files[0].label", equalTo("favicondataverse.png")) + .body("data.files[0].label", equalTo("dataverseproject.png")) .statusCode(OK.getStatusCode()); Long fileId = JsonPath.from(addResponse.body().asString()).getLong("data.files[0].dataFile.id"); @@ -62,7 +66,7 @@ public void testDownloadFile() { Response restrictResponse = UtilIT.restrictFile(fileId.toString(), restrict, apiToken); restrictResponse.prettyPrint(); restrictResponse.then().assertThat() - .body("data.message", equalTo("File favicondataverse.png restricted.")) + .body("data.message", equalTo("File dataverseproject.png restricted.")) .statusCode(OK.getStatusCode()); Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); @@ -146,6 +150,10 @@ public void testDownloadFile() { // Should get an OK response (able to download file) based on IP Group membership. No API token. assertEquals(OK.getStatusCode(), anonDownload.getStatusCode()); + Response collectionsResp = UtilIT.getUserPermittedCollections(username, apiToken, "DownloadFile"); + collectionsResp.prettyPrint(); + collectionsResp.then().assertThat() + .statusCode(OK.getStatusCode()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java index 6ac0957ae8e..791b890e92e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java @@ -552,8 +552,12 @@ public void testUserPermittedDataverses() { grantRoleResponse.then().assertThat() .statusCode(OK.getStatusCode()); - Response collectionsResp = UtilIT.getUserPermittedCollections(usernameOfUser, userApiToken, "ViewUnpublishedDataset"); + Response collectionsResp = UtilIT.getUserPermittedCollections(superuserUsername, userApiToken, "ViewUnpublishedDataset"); + collectionsResp.prettyPrint(); assertEquals(403, collectionsResp.getStatusCode()); + collectionsResp = UtilIT.getUserPermittedCollections(usernameOfUser, userApiToken, "ViewUnpublishedDataset"); + collectionsResp.prettyPrint(); + assertEquals(200, collectionsResp.getStatusCode()); collectionsResp = UtilIT.getUserPermittedCollections(usernameOfUser, "", "ViewUnpublishedDataset"); assertEquals(401, collectionsResp.getStatusCode()); collectionsResp = UtilIT.getUserPermittedCollections("fakeUser", superuserApiToken, "ViewUnpublishedDataset"); @@ -562,7 +566,7 @@ public void testUserPermittedDataverses() { assertEquals(500, collectionsResp.getStatusCode()); // Testing adding an explicit permission/role to one dataverse - collectionsResp = UtilIT.getUserPermittedCollections(usernameOfUser, superuserApiToken, "DownloadFile"); + collectionsResp = UtilIT.getUserPermittedCollections(usernameOfUser, userApiToken, "DownloadFile"); collectionsResp.prettyPrint(); collectionsResp.then().assertThat() .statusCode(OK.getStatusCode()) @@ -573,7 +577,7 @@ public void testUserPermittedDataverses() { assignRole.prettyPrint(); assertEquals(200, assignRole.getStatusCode()); - collectionsResp = UtilIT.getUserPermittedCollections(usernameOfUser, superuserApiToken, "DownloadFile"); + collectionsResp = UtilIT.getUserPermittedCollections(usernameOfUser, userApiToken, "DownloadFile"); collectionsResp.prettyPrint(); collectionsResp.then().assertThat() .statusCode(OK.getStatusCode()) @@ -585,7 +589,7 @@ public void testUserPermittedDataverses() { addToGroup.then().assertThat() .statusCode(OK.getStatusCode()); - collectionsResp = UtilIT.getUserPermittedCollections(usernameOfUser, superuserApiToken, "DownloadFile"); + collectionsResp = UtilIT.getUserPermittedCollections(usernameOfUser, userApiToken, "DownloadFile"); collectionsResp.prettyPrint(); collectionsResp.then().assertThat() .statusCode(OK.getStatusCode()) From ef853643a7dbbe0be0820b78d04d733234ad9061 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:43:36 -0400 Subject: [PATCH 0065/1048] updated docs --- .../6467-optimize-permission-lookups-for-a-user.md | 2 +- doc/sphinx-guides/source/api/native-api.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/6467-optimize-permission-lookups-for-a-user.md b/doc/release-notes/6467-optimize-permission-lookups-for-a-user.md index 7cc3f56d762..f0f01089870 100644 --- a/doc/release-notes/6467-optimize-permission-lookups-for-a-user.md +++ b/doc/release-notes/6467-optimize-permission-lookups-for-a-user.md @@ -4,6 +4,6 @@ The following API have been added: This API lists the dataverses/collections that the user has access to via the permission passed. By passing "any" as the permission the list will return all dataverse/collections that the user can access regardless of which permission is used. -This API can be executed only by superusers. +This API can be executed only by the User requesting their own list of accessible collections or by Administrators. Valid Permissions are: AddDataverse, AddDataset, ViewUnpublishedDataverse, ViewUnpublishedDataset, DownloadFile, EditDataverse, EditDataset, ManageDataversePermissions, ManageDatasetPermissions, ManageFilePermissions, PublishDataverse, PublishDataset, DeleteDataverse, DeleteDatasetDraft, and "any" as a wildcard option. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index da9823d81c2..8e9a9bf4be1 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -6019,7 +6019,7 @@ List Dataverse collections a user can act on based on a particular permission :: GET http://$SERVER/api/users/$identifier/allowedCollections/$permission -.. note:: This API can only be called by an Administrator +.. note:: This API can only be called by an Administrator or by a User requesting their own list of accessible collections. The ``$identifier`` is the username of the requested user. The ``$permission`` is the permission (tied to the roles) that gives the user access to the collection. From 529db683c50fa60f854bb2a3813212e2cc118370 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:49:50 -0400 Subject: [PATCH 0066/1048] update doc --- .../6467-optimize-permission-lookups-for-a-user.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/6467-optimize-permission-lookups-for-a-user.md b/doc/release-notes/6467-optimize-permission-lookups-for-a-user.md index f0f01089870..432ba0092f4 100644 --- a/doc/release-notes/6467-optimize-permission-lookups-for-a-user.md +++ b/doc/release-notes/6467-optimize-permission-lookups-for-a-user.md @@ -4,6 +4,6 @@ The following API have been added: This API lists the dataverses/collections that the user has access to via the permission passed. By passing "any" as the permission the list will return all dataverse/collections that the user can access regardless of which permission is used. -This API can be executed only by the User requesting their own list of accessible collections or by Administrators. +This API can be executed only by the User requesting their own list of accessible collections or by an Administrator. Valid Permissions are: AddDataverse, AddDataset, ViewUnpublishedDataverse, ViewUnpublishedDataset, DownloadFile, EditDataverse, EditDataset, ManageDataversePermissions, ManageDatasetPermissions, ManageFilePermissions, PublishDataverse, PublishDataset, DeleteDataverse, DeleteDatasetDraft, and "any" as a wildcard option. From 5a1f151e8f89dcbfa502d59386655ab294b6b3c2 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 8 Oct 2024 09:08:03 -0400 Subject: [PATCH 0067/1048] fix ip group check only for user asking for themself --- .../iq/dataverse/PermissionServiceBean.java | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index ee5e2aa9b55..1ee71232e3c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -919,24 +919,26 @@ public List findPermittedCollections(DataverseRequest request, Authen public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit) { if (user != null) { - // IP Group - IpAddress ip = request != null ? request.getSourceAddress() : new IPv4Address(0L); + // IP Group - Only check IP if a User is calling for themself String ipRangeSQL = "FALSE"; - if (ip instanceof IPv4Address) { - IPv4Address ipv4 = (IPv4Address) ip; - ipRangeSQL = ipv4.toBigInteger() + " BETWEEN ipv4range.bottomaslong AND ipv4range.topaslong"; - } else if (ip instanceof IPv6Address) { - IPv6Address ipv6 = (IPv6Address) ip; - long[] vals = ipv6.toLongArray(); - if (vals.length == 4) { - ipRangeSQL = """ - (@0 BETWEEN ipv6range.bottoma AND ipv6range.topa - AND @1 BETWEEN ipv6range.bottomb AND ipv6range.topb - AND @2 BETWEEN ipv6range.bottomc AND ipv6range.topc - AND @3 BETWEEN ipv6range.bottomd AND ipv6range.topd) - """; - for (int i = 0; i < vals.length; i++) { - ipRangeSQL = ipRangeSQL.replace("@" + i, String.valueOf(vals[i])); + if (request.getAuthenticatedUser().getUserIdentifier().equalsIgnoreCase(user.getUserIdentifier())) { + IpAddress ip = request != null ? request.getSourceAddress() : new IPv4Address(0L); + if (ip instanceof IPv4Address) { + IPv4Address ipv4 = (IPv4Address) ip; + ipRangeSQL = ipv4.toBigInteger() + " BETWEEN ipv4range.bottomaslong AND ipv4range.topaslong"; + } else if (ip instanceof IPv6Address) { + IPv6Address ipv6 = (IPv6Address) ip; + long[] vals = ipv6.toLongArray(); + if (vals.length == 4) { + ipRangeSQL = """ + (@0 BETWEEN ipv6range.bottoma AND ipv6range.topa + AND @1 BETWEEN ipv6range.bottomb AND ipv6range.topb + AND @2 BETWEEN ipv6range.bottomc AND ipv6range.topc + AND @3 BETWEEN ipv6range.bottomd AND ipv6range.topd) + """; + for (int i = 0; i < vals.length; i++) { + ipRangeSQL = ipRangeSQL.replace("@" + i, String.valueOf(vals[i])); + } } } } From a10ba683fd21369f62185ef8ea888c8aadc0f21f Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 8 Oct 2024 09:22:06 -0400 Subject: [PATCH 0068/1048] check for null request --- .../java/edu/harvard/iq/dataverse/PermissionServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index 1ee71232e3c..eeb10133078 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -921,7 +921,7 @@ public List findPermittedCollections(DataverseRequest request, Authen // IP Group - Only check IP if a User is calling for themself String ipRangeSQL = "FALSE"; - if (request.getAuthenticatedUser().getUserIdentifier().equalsIgnoreCase(user.getUserIdentifier())) { + if (request != null && request.getAuthenticatedUser().getUserIdentifier().equalsIgnoreCase(user.getUserIdentifier())) { IpAddress ip = request != null ? request.getSourceAddress() : new IPv4Address(0L); if (ip instanceof IPv4Address) { IPv4Address ipv4 = (IPv4Address) ip; From 6090fb5cd84d5b9480dbb7e08b35aea021610508 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:43:48 -0400 Subject: [PATCH 0069/1048] adding ipaddress to Sword --- .../harvard/iq/dataverse/PermissionServiceBean.java | 8 +++++--- .../api/datadeposit/SWORDv2ServiceDocumentServlet.java | 9 +++++++++ .../api/datadeposit/ServiceDocumentManagerImpl.java | 10 +++++++--- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index eeb10133078..772e0bb0fd6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -918,11 +918,13 @@ public List findPermittedCollections(DataverseRequest request, Authen } public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit) { if (user != null) { - // IP Group - Only check IP if a User is calling for themself String ipRangeSQL = "FALSE"; - if (request != null && request.getAuthenticatedUser().getUserIdentifier().equalsIgnoreCase(user.getUserIdentifier())) { - IpAddress ip = request != null ? request.getSourceAddress() : new IPv4Address(0L); + if (request != null + && request.getAuthenticatedUser() != null + && request.getSourceAddress() != null + && request.getAuthenticatedUser().getUserIdentifier().equalsIgnoreCase(user.getUserIdentifier())) { + IpAddress ip = request.getSourceAddress(); if (ip instanceof IPv4Address) { IPv4Address ipv4 = (IPv4Address) ip; ipRangeSQL = ipv4.toBigInteger() + " BETWEEN ipv4range.bottomaslong AND ipv4range.topaslong"; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SWORDv2ServiceDocumentServlet.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SWORDv2ServiceDocumentServlet.java index eab005d87fa..d5cd6759267 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SWORDv2ServiceDocumentServlet.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SWORDv2ServiceDocumentServlet.java @@ -1,6 +1,8 @@ package edu.harvard.iq.dataverse.api.datadeposit; import java.io.IOException; + +import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import jakarta.inject.Inject; import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; @@ -29,6 +31,13 @@ public void init() throws ServletException { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + String ipAddress = req.getHeader("X-FORWARDED-FOR"); + if (ipAddress == null) { + ipAddress = req.getRemoteAddr(); + } + if (ipAddress != null) { + serviceDocumentManagerImpl.setIpAddress(IpAddress.valueOf(ipAddress)); + } this.api.get(req, resp); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java index 423c631aade..62f23e97af9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java @@ -4,6 +4,7 @@ import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.PermissionServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.util.SystemConfig; @@ -38,6 +39,8 @@ public class ServiceDocumentManagerImpl implements ServiceDocumentManager { @Inject UrlManager urlManager; + private IpAddress ipAddress = null; + @Override public ServiceDocument getServiceDocument(String sdUri, AuthCredentials authCredentials, SwordConfiguration config) throws SwordError, SwordServerException, SwordAuthException { @@ -65,10 +68,8 @@ public ServiceDocument getServiceDocument(String sdUri, AuthCredentials authCred * a Shibboleth user can have an API token the transient * shibIdentityProvider String on AuthenticatedUser is only set when a * SAML assertion is made at runtime via the browser. - * - * We also don't support IP Groups since we don't have access to the request containing the ip address. */ - List dataverses = permissionService.findPermittedCollections(null, user, Permission.AddDataset); + List dataverses = permissionService.findPermittedCollections(new DataverseRequest(user, ipAddress), user, Permission.AddDataset); for (Dataverse dataverse : dataverses) { String dvAlias = dataverse.getAlias(); if (dvAlias != null && !dvAlias.isEmpty()) { @@ -85,4 +86,7 @@ public ServiceDocument getServiceDocument(String sdUri, AuthCredentials authCred return service; } + public void setIpAddress(IpAddress ipAddress) { + this.ipAddress = ipAddress; + } } From 3acd516c508eab8ff8b6f82fcb44f87a18e73176 Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Wed, 9 Oct 2024 15:41:22 +0200 Subject: [PATCH 0070/1048] Fixing null definitionPoint issue while setting up a new installation using setup-all.sh --- .../edu/harvard/iq/dataverse/DataverseRoleServiceBean.java | 6 ++++-- .../harvard/iq/dataverse/search/SolrIndexServiceBean.java | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java index 7ee6b7295a8..b751841da74 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java @@ -61,8 +61,10 @@ public DataverseRole save(DataverseRole aRole) { owner = dataverseService.findByAlias("root"); } - IndexResponse indexDefinitionPointResult = indexDefinitionPoint(owner); - logger.info("Indexing result: " + indexDefinitionPointResult); + if(owner != null) { // owner may be null if a role is created before the root collection as in setup-all.sh + IndexResponse indexDefinitionPointResult = indexDefinitionPoint(owner); + logger.info("Indexing result: " + indexDefinitionPointResult); + } return aRole; } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index cfe29ea08c7..a3d9afd3baa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -374,6 +374,12 @@ public IndexResponse indexPermissionsOnSelfAndChildren(long definitionPointId) { * inheritance */ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) { + + if (definitionPoint == null) { + logger.log(Level.WARNING, "Cannot perform indexPermissionsOnSelfAndChildren with a definitionPoint null"); + return null; + } + List dvObjectsToReindexPermissionsFor = new ArrayList<>(); List filesToReindexAsBatch = new ArrayList<>(); /** From be2c1432ea54da0fcf8f1332b0ee2bc95ae1b515 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 9 Oct 2024 11:40:45 -0400 Subject: [PATCH 0071/1048] change get ipaddress code --- .../api/datadeposit/SWORDv2ServiceDocumentServlet.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SWORDv2ServiceDocumentServlet.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SWORDv2ServiceDocumentServlet.java index d5cd6759267..a32e97bcee2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SWORDv2ServiceDocumentServlet.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SWORDv2ServiceDocumentServlet.java @@ -2,7 +2,7 @@ import java.io.IOException; -import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import jakarta.inject.Inject; import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; @@ -31,13 +31,7 @@ public void init() throws ServletException { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - String ipAddress = req.getHeader("X-FORWARDED-FOR"); - if (ipAddress == null) { - ipAddress = req.getRemoteAddr(); - } - if (ipAddress != null) { - serviceDocumentManagerImpl.setIpAddress(IpAddress.valueOf(ipAddress)); - } + serviceDocumentManagerImpl.setIpAddress((new DataverseRequest(null, req)).getSourceAddress()); this.api.get(req, resp); } From 2046950e4d93ce988aaee2e1e178a9d2ba5b5c13 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 10 Oct 2024 17:04:58 -0400 Subject: [PATCH 0072/1048] more solr version changes --- .../10713-Solr9.6.1 and lib updates.md | 10 ++++++++++ .../_static/installation/files/etc/init.d/solr | 2 +- .../installation/files/etc/systemd/solr.service | 6 +++--- .../source/developers/classic-dev-env.rst | 2 +- .../source/installation/prerequisites.rst | 16 ++++++++-------- docker/compose/demo/compose.yml | 2 +- 6 files changed, 24 insertions(+), 14 deletions(-) create mode 100644 doc/release-notes/10713-Solr9.6.1 and lib updates.md diff --git a/doc/release-notes/10713-Solr9.6.1 and lib updates.md b/doc/release-notes/10713-Solr9.6.1 and lib updates.md new file mode 100644 index 00000000000..84807f6c176 --- /dev/null +++ b/doc/release-notes/10713-Solr9.6.1 and lib updates.md @@ -0,0 +1,10 @@ +Solr 9.6.1 is now the version recommended in our installation guides and used with automated testing. Other libraries Dataverse uses have been updated as well. + +For the upgrade instructions section: + +[note that 6.5 may contain other solr-related changes, so the instructions may need to contain information merged from multiple release notes!] + +If you are upgrading Solr: + - Install solr-9.6.1 following the instructions from the Installation guide. + - Run a full reindex to populate the search catalog. + - Note that it may be possible to skip the reindexing step by simply moving the existing `.../server/solr/collection1/` under the new `solr-9.6.1` installation directory. This however has not been thoroughly tested and is not officially supported. \ No newline at end of file diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr index 14df734cca7..4326ca70aaf 100755 --- a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr +++ b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr @@ -5,7 +5,7 @@ # chkconfig: 35 92 08 # description: Starts and stops Apache Solr -SOLR_DIR="/usr/local/solr/solr-9.4.1" +SOLR_DIR="/usr/local/solr/solr-9.6.1" SOLR_COMMAND="bin/solr" SOLR_ARGS="-m 1g" SOLR_USER=solr diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service index 8ccf7652a49..8aee5db3d4d 100644 --- a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service +++ b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service @@ -5,9 +5,9 @@ After = syslog.target network.target remote-fs.target nss-lookup.target [Service] User = solr Type = forking -WorkingDirectory = /usr/local/solr/solr-9.4.1 -ExecStart = /usr/local/solr/solr-9.4.1/bin/solr start -m 1g -ExecStop = /usr/local/solr/solr-9.4.1/bin/solr stop +WorkingDirectory = /usr/local/solr/solr-9.6.1 +ExecStart = /usr/local/solr/solr-9.6.1/bin/solr start -m 1g +ExecStop = /usr/local/solr/solr-9.6.1/bin/solr stop LimitNOFILE=65000 LimitNPROC=65000 Restart=on-failure diff --git a/doc/sphinx-guides/source/developers/classic-dev-env.rst b/doc/sphinx-guides/source/developers/classic-dev-env.rst index d305019004e..3806a791fff 100755 --- a/doc/sphinx-guides/source/developers/classic-dev-env.rst +++ b/doc/sphinx-guides/source/developers/classic-dev-env.rst @@ -136,7 +136,7 @@ On Linux, you should just install PostgreSQL using your favorite package manager Install Solr ^^^^^^^^^^^^ -`Solr `_ 9.4.1 is required. +`Solr `_ 9.6.1 is required. Follow the instructions in the "Installing Solr" section of :doc:`/installation/prerequisites` in the main Installation guide. diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index f61321ef245..48d0158e740 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -163,7 +163,7 @@ The Dataverse software search index is powered by Solr. Supported Versions ================== -The Dataverse software has been tested with Solr version 9.4.1. Future releases in the 9.x series are likely to be compatible. Please get in touch (:ref:`support`) if you are having trouble with a newer version. +The Dataverse software has been tested with Solr version 9.6.1. Future releases in the 9.x series are likely to be compatible. Please get in touch (:ref:`support`) if you are having trouble with a newer version. Installing Solr =============== @@ -178,19 +178,19 @@ Become the ``solr`` user and then download and configure Solr:: su - solr cd /usr/local/solr - wget https://archive.apache.org/dist/solr/solr/9.4.1/solr-9.4.1.tgz - tar xvzf solr-9.4.1.tgz - cd solr-9.4.1 + wget https://archive.apache.org/dist/solr/solr/9.6.1/solr-9.6.1.tgz + tar xvzf solr-9.6.1.tgz + cd solr-9.6.1 cp -r server/solr/configsets/_default server/solr/collection1 You should already have a "dvinstall.zip" file that you downloaded from https://github.com/IQSS/dataverse/releases . Unzip it into ``/tmp``. Then copy the files into place:: - cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf - cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf + cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-9.6.1/server/solr/collection1/conf + cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-9.6.1/server/solr/collection1/conf Note: The Dataverse Project team has customized Solr to boost results that come from certain indexed elements inside the Dataverse installation, for example prioritizing results from Dataverse collections over Datasets. If you would like to remove this, edit your ``solrconfig.xml`` and remove the ```` element and its contents. If you have ideas about how this boosting could be improved, feel free to contact us through our Google Group https://groups.google.com/forum/#!forum/dataverse-dev . -A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-9.4.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400`` +A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-9.6.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400`` Solr will warn about needing to increase the number of file descriptors and max processes in a production environment but will still run with defaults. We have increased these values to the recommended levels by adding ulimit -n 65000 to the init script, and the following to ``/etc/security/limits.conf``:: @@ -209,7 +209,7 @@ Solr launches asynchronously and attempts to use the ``lsof`` binary to watch fo Finally, you need to tell Solr to create the core "collection1" on startup:: - echo "name=collection1" > /usr/local/solr/solr-9.4.1/server/solr/collection1/core.properties + echo "name=collection1" > /usr/local/solr/solr-9.6.1/server/solr/collection1/core.properties Dataverse collection ("dataverse") page uses Solr very heavily. On a busy instance this may cause the search engine to become the performance bottleneck, making these pages take increasingly longer to load, potentially affecting the overall performance of the application and/or causing Solr itself to crash. If this is observed on your instance, we recommend uncommenting the following lines in the ```` section of the ``solrconfig.xml`` file:: diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index 33e7b52004b..19f4440081a 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -103,7 +103,7 @@ services: solr: container_name: "solr" hostname: "solr" - image: solr:9.4.1 + image: solr:9.6.1 depends_on: - solr_initializer restart: on-failure From cd63c39b0513dc8bb85d9b452cbaf4a6dc5e7500 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 10 Oct 2024 17:25:49 -0400 Subject: [PATCH 0073/1048] up solr version --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index d5cffcec0aa..ccabcca2b02 100644 --- a/.env +++ b/.env @@ -1,5 +1,5 @@ APP_IMAGE=gdcc/dataverse:unstable POSTGRES_VERSION=16 DATAVERSE_DB_USER=dataverse -SOLR_VERSION=9.3.0 +SOLR_VERSION=9.6.1 SKIP_DEPLOY=0 \ No newline at end of file From 42f64cb2e11ff73f8d1668ffa58219e15f19a84d Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Thu, 17 Oct 2024 14:28:20 +0200 Subject: [PATCH 0074/1048] test: add test for range search queries for ints, floats and dates --- .../harvard/iq/dataverse/api/SearchIT.java | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index 3a2b684c421..8850b7ce7c2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -1269,6 +1269,199 @@ public void testGeospatialSearchInvalid() { } + @Test + public void testRangeQueries() { + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + // Using the "astrophysics" block because it contains all field types relevant for range queries + // (int, float and date) + Response setMetadataBlocks = UtilIT.setMetadataBlocks(dataverseAlias, Json.createArrayBuilder().add("citation").add("astrophysics"), apiToken); + setMetadataBlocks.prettyPrint(); + setMetadataBlocks.then().assertThat().statusCode(OK.getStatusCode()); + + JsonObjectBuilder datasetJson = Json.createObjectBuilder() + .add("datasetVersion", Json.createObjectBuilder() + .add("metadataBlocks", Json.createObjectBuilder() + .add("citation", Json.createObjectBuilder() + .add("fields", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("typeName", "title") + .add("value", "Test Astrophysics Dataset") + .add("typeClass", "primitive") + .add("multiple", false) + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("authorName", + Json.createObjectBuilder() + .add("value", "Simpson, Homer") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "authorName")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "author") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("datasetContactEmail", + Json.createObjectBuilder() + .add("value", "hsimpson@mailinator.com") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "datasetContactEmail")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "datasetContact") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("dsDescriptionValue", + Json.createObjectBuilder() + .add("value", "This is a test dataset.") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "dsDescriptionValue")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "dsDescription") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add("Other") + ) + .add("typeClass", "controlledVocabulary") + .add("multiple", true) + .add("typeName", "subject") + ) + ) + ) + .add("astrophysics", Json.createObjectBuilder() + .add("fields", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("typeName", "coverage.Temporal") + .add("typeClass", "compound") + .add("multiple", true) + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("coverage.Temporal.StartTime", + Json.createObjectBuilder() + .add("value", "2015-01-01") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "coverage.Temporal.StartTime") + ) + ) + ) + ) + .add(Json.createObjectBuilder() + .add("typeName", "coverage.ObjectCount") + .add("typeClass", "primitive") + .add("multiple", false) + .add("value", "9000") + ) + .add(Json.createObjectBuilder() + .add("typeName", "coverage.SkyFraction") + .add("typeClass", "primitive") + .add("multiple", false) + .add("value", "0.002") + ) + ) + ) + )); + + Response createDatasetResponse = UtilIT.createDataset(dataverseAlias, datasetJson, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + String datasetPid = JsonPath.from(createDatasetResponse.getBody().asString()).getString("data.persistentId"); + + // Integer range query: Hit + Response search1 = UtilIT.search("id:dataset_" + datasetId + "_draft AND coverage.ObjectCount:[1000 TO 10000]", apiToken, "&show_entity_ids=true"); + search1.prettyPrint(); + search1.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.total_count", CoreMatchers.is(1)) + .body("data.count_in_response", CoreMatchers.is(1)) + .body("data.items[0].entity_id", CoreMatchers.is(datasetId)); + + // Integer range query: Miss + Response search2 = UtilIT.search("id:dataset_" + datasetId + "_draft AND coverage.ObjectCount:[* TO 1000]", apiToken); + search2.prettyPrint(); + search2.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.total_count", CoreMatchers.is(0)) + .body("data.count_in_response", CoreMatchers.is(0)); + + // Float range query: Hit + Response search3 = UtilIT.search("id:dataset_" + datasetId + "_draft AND coverage.SkyFraction:[0 TO 0.5]", apiToken, "&show_entity_ids=true"); + search3.prettyPrint(); + search3.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.total_count", CoreMatchers.is(1)) + .body("data.count_in_response", CoreMatchers.is(1)) + .body("data.items[0].entity_id", CoreMatchers.is(datasetId)); + + // Float range query: Miss + Response search4 = UtilIT.search("id:dataset_" + datasetId + "_draft AND coverage.SkyFraction:[0.5 TO 1]", apiToken); + search4.prettyPrint(); + search4.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.total_count", CoreMatchers.is(0)) + .body("data.count_in_response", CoreMatchers.is(0)); + + // Date range query: Hit + Response search5 = UtilIT.search("id:dataset_" + datasetId + "_draft AND coverage.Temporal.StartTime:2015", apiToken, "&show_entity_ids=true"); + search5.prettyPrint(); + search5.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.total_count", CoreMatchers.is(1)) + .body("data.count_in_response", CoreMatchers.is(1)) + .body("data.items[0].entity_id", CoreMatchers.is(datasetId)); + + // Date range query: Miss + Response search6 = UtilIT.search("id:dataset_" + datasetId + "_draft AND coverage.Temporal.StartTime:[2020 TO *]", apiToken); + search6.prettyPrint(); + search6.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.total_count", CoreMatchers.is(0)) + .body("data.count_in_response", CoreMatchers.is(0)); + + // Combining all three range queries: Hit + Response search7 = UtilIT.search("id:dataset_" + datasetId + "_draft AND coverage.ObjectCount:[1000 TO 10000] AND coverage.SkyFraction:[0 TO 0.5] AND coverage.Temporal.StartTime:2015", apiToken, "&show_entity_ids=true"); + search7.prettyPrint(); + search7.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.total_count", CoreMatchers.is(1)) + .body("data.count_in_response", CoreMatchers.is(1)) + .body("data.items[0].entity_id", CoreMatchers.is(datasetId)); + + // Combining all three range queries: Miss + Response search8 = UtilIT.search("id:dataset_" + datasetId + "_draft AND coverage.ObjectCount:[* TO 1000] AND coverage.SkyFraction:[0.5 TO 1] AND coverage.Temporal.StartTime:[2020 TO *]", apiToken); + search8.prettyPrint(); + search8.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.total_count", CoreMatchers.is(0)) + .body("data.count_in_response", CoreMatchers.is(0)); + + } + @AfterEach public void tearDownDataverse() { File treesThumb = new File("scripts/search/data/binary/trees.png.thumb48"); From ed7e38ec57e302f2a40f7491b3360878c6eb187b Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Fri, 18 Oct 2024 15:59:18 +0200 Subject: [PATCH 0075/1048] feat: skip indexing of field instead of entire dataset when encountering invalid ints, floats or dates --- .../iq/dataverse/search/IndexServiceBean.java | 102 ++++++++++---- .../harvard/iq/dataverse/api/SearchIT.java | 128 ++++++++++++++++++ 2 files changed, 207 insertions(+), 23 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index e73b8d2f679..17dc6726a5a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -27,6 +27,8 @@ import java.sql.Timestamp; import java.text.SimpleDateFormat; import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; @@ -44,6 +46,7 @@ import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.regex.Pattern; import java.util.stream.Collectors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; @@ -1060,36 +1063,89 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set indexableValues = dsf.getValuesWithoutNaValues().stream() + .filter(s -> intPattern.matcher(s).find()) + .collect(Collectors.toList()); + solrInputDocument.addField(solrFieldSearchable, indexableValues); + if (dsfType.getSolrField().isFacetable()) { + solrInputDocument.addField(solrFieldFacetable, indexableValues); + } + } else if (dsfType.getSolrField().getSolrType().equals(SolrField.SolrType.FLOAT)) { + // same as for integer values, we need to filter invalid float values + List indexableValues = dsf.getValuesWithoutNaValues().stream() + .filter(s -> { + try { + Double.parseDouble(s); + return true; + } catch (NumberFormatException e) { + return false; + } + }) + .collect(Collectors.toList()); + solrInputDocument.addField(solrFieldSearchable, indexableValues); + if (dsfType.getSolrField().isFacetable()) { + solrInputDocument.addField(solrFieldFacetable, indexableValues); + } } else if (dsfType.getSolrField().getSolrType().equals(SolrField.SolrType.DATE)) { - // we index dates as full strings (YYYY, YYYY-MM or YYYY-MM-DD) - // for use in facets, we index only the year (YYYY) + // Solr accepts dates in the ISO-8601 format, e.g. YYYY-MM-DDThh:mm:ssZ, YYYYY-MM-DD, YYYY-MM, YYYY + // See: https://solr.apache.org/guide/solr/latest/indexing-guide/date-formatting-math.html + // If dates have been entered in other formats, we need to skip or convert them + // TODO at the moment we are simply skipping, but converting them would offer more value for search + // For use in facets, we index only the year (YYYY) String dateAsString = ""; if (!dsf.getValues_nondisplay().isEmpty()) { - dateAsString = dsf.getValues_nondisplay().get(0); - } + dateAsString = dsf.getValues_nondisplay().get(0).trim(); + } + logger.fine("date as string: " + dateAsString); + if (dateAsString != null && !dateAsString.isEmpty()) { - SimpleDateFormat inputDateyyyy = new SimpleDateFormat("yyyy", Locale.ENGLISH); - try { - /** - * @todo when bean validation is working we - * won't have to convert strings into dates - */ - logger.fine("Trying to convert " + dateAsString + " to a YYYY date from dataset " + dataset.getId()); - Date dateAsDate = inputDateyyyy.parse(dateAsString); - SimpleDateFormat yearOnly = new SimpleDateFormat("yyyy"); - String datasetFieldFlaggedAsDate = yearOnly.format(dateAsDate); - logger.fine("YYYY only: " + datasetFieldFlaggedAsDate); - // solrInputDocument.addField(solrFieldSearchable, - // Integer.parseInt(datasetFieldFlaggedAsDate)); - solrInputDocument.addField(solrFieldSearchable, dateAsString); - if (dsfType.getSolrField().isFacetable()) { - // solrInputDocument.addField(solrFieldFacetable, + boolean dateValid = false; + + DateTimeFormatter[] possibleFormats = { + DateTimeFormatter.ISO_INSTANT, + DateTimeFormatter.ofPattern("yyyy-MM-dd"), + DateTimeFormatter.ofPattern("yyyy-MM"), + DateTimeFormatter.ofPattern("yyyy") + }; + for (DateTimeFormatter format : possibleFormats){ + try { + format.parse(dateAsString); + dateValid = true; + } catch (DateTimeParseException e) { + // no-op, date is invalid + } + } + + if (!dateValid) { + logger.fine("couldn't index " + dsf.getDatasetFieldType().getName() + ":" + dsf.getValues() + " because it's not a valid date format according to Solr"); + } else { + SimpleDateFormat inputDateyyyy = new SimpleDateFormat("yyyy", Locale.ENGLISH); + try { + /** + * @todo when bean validation is working we + * won't have to convert strings into dates + */ + logger.fine("Trying to convert " + dateAsString + " to a YYYY date from dataset " + dataset.getId()); + Date dateAsDate = inputDateyyyy.parse(dateAsString); + SimpleDateFormat yearOnly = new SimpleDateFormat("yyyy"); + String datasetFieldFlaggedAsDate = yearOnly.format(dateAsDate); + logger.fine("YYYY only: " + datasetFieldFlaggedAsDate); + // solrInputDocument.addField(solrFieldSearchable, // Integer.parseInt(datasetFieldFlaggedAsDate)); - solrInputDocument.addField(solrFieldFacetable, datasetFieldFlaggedAsDate); + solrInputDocument.addField(solrFieldSearchable, dateAsString); + if (dsfType.getSolrField().isFacetable()) { + // solrInputDocument.addField(solrFieldFacetable, + // Integer.parseInt(datasetFieldFlaggedAsDate)); + solrInputDocument.addField(solrFieldFacetable, datasetFieldFlaggedAsDate); + } + } catch (Exception ex) { + logger.info("unable to convert " + dateAsString + " into YYYY format and couldn't index it (" + dsfType.getName() + ")"); } - } catch (Exception ex) { - logger.info("unable to convert " + dateAsString + " into YYYY format and couldn't index it (" + dsfType.getName() + ")"); } } } else { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index 8850b7ce7c2..6058ab17d72 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -1462,6 +1462,134 @@ public void testRangeQueries() { } + @Test + public void testSearchWithInvalidDateField() { + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response setMetadataBlocks = UtilIT.setMetadataBlocks(dataverseAlias, Json.createArrayBuilder().add("citation"), apiToken); + setMetadataBlocks.prettyPrint(); + setMetadataBlocks.then().assertThat().statusCode(OK.getStatusCode()); + + // Adding a dataset with a date in the "timePeriodCoveredStart" field that doesn't match Solr's date format + // (ISO-8601 format, e.g. YYYY-MM-DDThh:mm:ssZ, YYYYY-MM-DD, YYYY-MM, YYYY) + // (See: https://solr.apache.org/guide/solr/latest/indexing-guide/date-formatting-math.html) + // So the date currently cannot be indexed + JsonObjectBuilder datasetJson = Json.createObjectBuilder() + .add("datasetVersion", Json.createObjectBuilder() + .add("metadataBlocks", Json.createObjectBuilder() + .add("citation", Json.createObjectBuilder() + .add("fields", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("typeName", "title") + .add("value", "Test Dataset") + .add("typeClass", "primitive") + .add("multiple", false) + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("authorName", + Json.createObjectBuilder() + .add("value", "Simpson, Homer") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "authorName")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "author") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("datasetContactEmail", + Json.createObjectBuilder() + .add("value", "hsimpson@mailinator.com") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "datasetContactEmail")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "datasetContact") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("dsDescriptionValue", + Json.createObjectBuilder() + .add("value", "This is a test dataset.") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "dsDescriptionValue")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "dsDescription") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add("Other") + ) + .add("typeClass", "controlledVocabulary") + .add("multiple", true) + .add("typeName", "subject") + ) + .add(Json.createObjectBuilder() + .add("typeName", "timePeriodCovered") + .add("typeClass", "compound") + .add("multiple", true) + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("timePeriodCoveredStart", + Json.createObjectBuilder() + .add("value", "15-01-01") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "timePeriodCoveredStart") + ) + ) + ) + ) + ) + ) + )); + + Response createDatasetResponse = UtilIT.createDataset(dataverseAlias, datasetJson, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + String datasetPid = JsonPath.from(createDatasetResponse.getBody().asString()).getString("data.persistentId"); + + // When querying on the date field: miss (because the date field was skipped during indexing) + Response search1 = UtilIT.search("id:dataset_" + datasetId + "_draft AND timePeriodCoveredStart:[2000 TO 2020]", apiToken); + search1.prettyPrint(); + search1.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.total_count", CoreMatchers.is(0)) + .body("data.count_in_response", CoreMatchers.is(0)); + + // When querying not on the date field: the dataset can be found (only the date field was skipped during indexing, not the entire dataset) + Response search2 = UtilIT.search("id:dataset_" + datasetId + "_draft", apiToken, "&show_entity_ids=true"); + search2.prettyPrint(); + search2.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.total_count", CoreMatchers.is(1)) + .body("data.count_in_response", CoreMatchers.is(1)) + .body("data.items[0].entity_id", CoreMatchers.is(datasetId)); + + } + @AfterEach public void tearDownDataverse() { File treesThumb = new File("scripts/search/data/binary/trees.png.thumb48"); From 3ec09cae2cd16e6e53e928c8ee20c46c9d26028e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 18 Oct 2024 15:36:05 -0400 Subject: [PATCH 0076/1048] initial pom csl entries --- pom.xml | 57 ++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/pom.xml b/pom.xml index edf72067976..6918005954c 100644 --- a/pom.xml +++ b/pom.xml @@ -709,6 +709,21 @@ hazelcast test + + de.undercouch + citeproc-java + 3.1.0 + + + org.citationstyles + styles + 24.3 + + + org.citationstyles + locales + 24.3 + @@ -743,7 +758,7 @@ src/main/resources - + true **/*.properties @@ -756,7 +771,7 @@ maven-compiler-plugin ${target.java.version} - + ${compilerArgument} @@ -909,14 +924,14 @@ org.apache.maven.plugins maven-surefire-plugin - - + + ${testsToExclude} ${skipUnitTests} ${surefire.jacoco.args} ${argLine} - + org.apache.maven.plugins maven-failsafe-plugin @@ -952,7 +967,7 @@ generate-schema - + process-classes ${openapi.outputDirectory} @@ -973,8 +988,8 @@ dev - - + + true @@ -984,14 +999,14 @@ all-unit-tests - + ct - + true true - + docker-build 16 @@ -999,16 +1014,16 @@ unstable false gdcc/base:${base.image.tag} - + noble - + ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} gdcc/configbaker:${conf.image.tag} ${app.image.tag} - - + + ${app.image} ${postgresql.server.version} ${solr.version} @@ -1018,7 +1033,7 @@ - + org.apache.maven.plugins maven-war-plugin @@ -1033,15 +1048,15 @@ - - + + io.fabric8 docker-maven-plugin true - + dev_dataverse ${app.image} @@ -1089,8 +1104,8 @@ ${project.basedir}/modules/container-configbaker/assembly.xml - - org.junit.jupiter @@ -709,21 +724,6 @@ hazelcast test - - de.undercouch - citeproc-java - 3.1.0 - - - org.citationstyles - styles - 24.3 - - - org.citationstyles - locales - 24.3 - diff --git a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java index c1197908a4d..9ac1c6170ff 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java @@ -36,6 +36,13 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.DateUtil; import org.apache.commons.text.StringEscapeUtils; + +import de.undercouch.citeproc.CSL; +import de.undercouch.citeproc.csl.CSLItemData; +import de.undercouch.citeproc.csl.CSLItemDataBuilder; +import de.undercouch.citeproc.csl.CSLNameBuilder; +import de.undercouch.citeproc.csl.CSLType; + import org.apache.commons.lang3.StringUtils; /** @@ -650,28 +657,21 @@ public Map getDataCiteMetadata() { metadata.put("datacite.publisher", producerString); metadata.put("datacite.publicationyear", getYear()); return metadata; - } - - String getCSLFormat(String style) { - CSLItemData item = new CSLItemDataBuilder() - .type(CSLType.ARTICLE_JOURNAL) - .title("Protein measurement with the Folin phenol reagent") - .author( - new CSLNameBuilder().given("Oliver H.").family("Lowry").build(), - new CSLNameBuilder().given("Nira J.").family("Rosebrough").build(), - new CSLNameBuilder().given("A. Lewis").family("Farr").build(), - new CSLNameBuilder().given("Rose J.").family("Randall").build() - ) - .issued(1951) - .containerTitle("The Journal of biological chemistry") - .volume(193) - .issue(1) - .page(265, 275) - .build(); - - return CSL.makeAdhocBibliography("apa", item).makeString(); - } - + } + + String getCSLFormat(String style) throws IOException { + CSLItemData item = new CSLItemDataBuilder().type(CSLType.ARTICLE_JOURNAL) + .title("Protein measurement with the Folin phenol reagent") + .author(new CSLNameBuilder().given("Oliver H.").family("Lowry").build(), + new CSLNameBuilder().given("Nira J.").family("Rosebrough").build(), + new CSLNameBuilder().given("A. Lewis").family("Farr").build(), + new CSLNameBuilder().given("Rose J.").family("Randall").build()) + .issued(1951).containerTitle("The Journal of biological chemistry").volume(193).issue(1).page(265, 275) + .build(); + + return CSL.makeAdhocBibliography("apa", item).makeString(); + } + // helper methods private String formatString(String value, boolean escapeHtml) { return formatString(value, escapeHtml, ""); diff --git a/src/main/webapp/dataset-citation.xhtml b/src/main/webapp/dataset-citation.xhtml index 346e39dc463..3ec301d71bc 100644 --- a/src/main/webapp/dataset-citation.xhtml +++ b/src/main/webapp/dataset-citation.xhtml @@ -41,6 +41,11 @@
    • +
  • + +
    • diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 6de0f00e94e..5fdbf959c1d 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1971,6 +1971,18 @@
    + + +
    + Link +
    +
    + +
    +     + From ffc8abfb1b74445ee16b9019d8acb7a6a7a4274f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 7 Nov 2024 14:05:27 -0500 Subject: [PATCH 0122/1048] fix script path, fix asURL --- src/main/webapp/dataset.xhtml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 5fdbf959c1d..1e4f7e622da 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1973,9 +1973,11 @@ -
    - Link -
    + + +
    + Link +
    + + +

    + + + + +

    +
    + + +
    +
    + + +
    +     +     From 4407cb756a61c52bd6fdb021f1fa8f5a0928f7c4 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 1 Nov 2024 11:55:24 -0400 Subject: [PATCH 0143/1048] typo on hides --- src/main/webapp/dataset.xhtml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 93a06cba5ad..d1d21bccc5b 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -2001,9 +2001,9 @@
    - From 3de69562324e6f8721a2cf3fa1b63dce7434ded0 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 1 Nov 2024 11:58:27 -0400 Subject: [PATCH 0144/1048] Add separate method/success msg --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 4 ++++ src/main/java/propertyFiles/Bundle.properties | 1 + 2 files changed, 5 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 8522f2733c7..053263051c5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -6737,5 +6737,9 @@ public String getSignpostingLinkHeader() { public boolean isDOI() { return AbstractDOIProvider.DOI_PROTOCOL.equals(dataset.getGlobalId().getProtocol()); } + + public void saveCreationNote() { + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.creationNoteSuccess")); + } } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 7623a7b5ba0..db0f54dbbd5 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1669,6 +1669,7 @@ dataset.message.createFailure=The dataset could not be created. dataset.message.termsFailure=The dataset terms could not be updated. dataset.message.label.fileAccess=Publicly-accessible storage dataset.message.publicInstall=Files in this dataset may be readable outside Dataverse, restricted and embargoed access are disabled +dataset.message.creationNoteSuccess=Creation note successfully updated. dataset.metadata.publicationDate=Publication Date dataset.metadata.publicationDate.tip=The publication date of a Dataset. dataset.metadata.citationDate=Citation Date From 1e8da09f9d71b850eb4fbf50d2c5cc22a574fc2b Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 1 Nov 2024 12:34:02 -0400 Subject: [PATCH 0145/1048] add edit in table --- src/main/java/propertyFiles/Bundle.properties | 1 + src/main/webapp/dataset-versions.xhtml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index db0f54dbbd5..1580392c101 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2051,6 +2051,7 @@ file.dataFilesTab.versions.description.firstPublished=This is the first publishe file.dataFilesTab.versions.description.deaccessionedReason=Deaccessioned Reason: file.dataFilesTab.versions.description.beAccessedAt=The dataset can now be accessed at: file.dataFilesTab.versions.viewDetails.btn=View Details +file.dataFilesTab.versions.creationNote.edit.btn=Edit Creation Note file.dataFilesTab.versions.widget.viewMoreInfo=To view more information about the versions of this dataset, and to edit it if this is your dataset, please visit the full version of this dataset at the {2}. file.dataFilesTab.versions.preloadmessage=(Loading versions...) file.previewTab.externalTools.header=Available Previews diff --git a/src/main/webapp/dataset-versions.xhtml b/src/main/webapp/dataset-versions.xhtml index 5c8680ab3fc..7fcbdbb52ca 100644 --- a/src/main/webapp/dataset-versions.xhtml +++ b/src/main/webapp/dataset-versions.xhtml @@ -135,6 +135,9 @@ + From db08d5a0a2f6cf12177f6f5a5946bf28ba70d4c8 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 1 Nov 2024 12:34:13 -0400 Subject: [PATCH 0146/1048] call save again --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 053263051c5..8250f9d628b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -183,7 +183,7 @@ public class DatasetPage implements java.io.Serializable { public enum EditMode { - CREATE, INFO, FILE, METADATA, LICENSE + CREATE, INFO, FILE, METADATA, LICENSE, CREATIONNOTE }; public enum DisplayMode { @@ -4079,8 +4079,9 @@ public String save() { } if (editMode.equals(EditMode.FILE)) { JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.filesSuccess")); + } if (editMode.equals(EditMode.CREATIONNOTE)) { + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.creationNoteSuccess")); } - } else { // must have been a bulk file update or delete: if (bulkFileDeleteInProgress) { @@ -6739,7 +6740,7 @@ public boolean isDOI() { } public void saveCreationNote() { - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.creationNoteSuccess")); + this.editMode=EditMode.CREATIONNOTE; } } From f529c51680afe6d4cbd469fc7bbc61ca73546dd2 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 1 Nov 2024 12:43:16 -0400 Subject: [PATCH 0147/1048] typo --- src/main/java/propertyFiles/Bundle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 1580392c101..c7f0cbdaf83 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2051,7 +2051,7 @@ file.dataFilesTab.versions.description.firstPublished=This is the first publishe file.dataFilesTab.versions.description.deaccessionedReason=Deaccessioned Reason: file.dataFilesTab.versions.description.beAccessedAt=The dataset can now be accessed at: file.dataFilesTab.versions.viewDetails.btn=View Details -file.dataFilesTab.versions.creationNote.edit.btn=Edit Creation Note +file.dataFilesTab.versions.creationNote.btn=Edit Creation Note file.dataFilesTab.versions.widget.viewMoreInfo=To view more information about the versions of this dataset, and to edit it if this is your dataset, please visit the full version of this dataset at the {2}. file.dataFilesTab.versions.preloadmessage=(Loading versions...) file.previewTab.externalTools.header=Available Previews From f6b8a98c377e771d2a947ee8b0aba1d97e219baf Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 1 Nov 2024 12:45:20 -0400 Subject: [PATCH 0148/1048] really add save --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 8250f9d628b..d111dcc3663 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -6741,6 +6741,7 @@ public boolean isDOI() { public void saveCreationNote() { this.editMode=EditMode.CREATIONNOTE; + save(); } } From fbda72ec920039e001dcad7eba7daa76130a066c Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 1 Nov 2024 13:45:01 -0400 Subject: [PATCH 0149/1048] pass workingVersion, only allow edit when on draft page --- src/main/webapp/dataset-versions.xhtml | 2 +- src/main/webapp/dataset.xhtml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/dataset-versions.xhtml b/src/main/webapp/dataset-versions.xhtml index 7fcbdbb52ca..aac800e2079 100644 --- a/src/main/webapp/dataset-versions.xhtml +++ b/src/main/webapp/dataset-versions.xhtml @@ -135,7 +135,7 @@ - diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index d1d21bccc5b..c60dda14104 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -984,6 +984,7 @@ + From 13d181557bc4029252822a14d0c266fd7883a55f Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 1 Nov 2024 13:47:14 -0400 Subject: [PATCH 0150/1048] shorten name --- src/main/java/propertyFiles/Bundle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index c7f0cbdaf83..2498b3bff0e 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2051,7 +2051,7 @@ file.dataFilesTab.versions.description.firstPublished=This is the first publishe file.dataFilesTab.versions.description.deaccessionedReason=Deaccessioned Reason: file.dataFilesTab.versions.description.beAccessedAt=The dataset can now be accessed at: file.dataFilesTab.versions.viewDetails.btn=View Details -file.dataFilesTab.versions.creationNote.btn=Edit Creation Note +file.dataFilesTab.versions.creationNote.btn=Edit Note file.dataFilesTab.versions.widget.viewMoreInfo=To view more information about the versions of this dataset, and to edit it if this is your dataset, please visit the full version of this dataset at the {2}. file.dataFilesTab.versions.preloadmessage=(Loading versions...) file.previewTab.externalTools.header=Available Previews From 41232ac8d439fbc85c538c6ab6adc2d6300a08ca Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 1 Nov 2024 13:47:45 -0400 Subject: [PATCH 0151/1048] Add div to cause button to be on next line --- src/main/webapp/dataset-versions.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/dataset-versions.xhtml b/src/main/webapp/dataset-versions.xhtml index aac800e2079..b024f23b602 100644 --- a/src/main/webapp/dataset-versions.xhtml +++ b/src/main/webapp/dataset-versions.xhtml @@ -133,7 +133,7 @@ - +
    Date: Fri, 1 Nov 2024 14:56:26 -0400 Subject: [PATCH 0152/1048] add note to json, ddi, datacite --- .../dataverse/api/dto/DatasetVersionDTO.java | 20 ++++++++++--------- .../dataverse/export/ddi/DdiExportUtil.java | 4 ++++ .../pidproviders/doi/XmlMetadataTemplate.java | 8 +++++++- .../iq/dataverse/util/json/JsonPrinter.java | 3 ++- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetVersionDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetVersionDTO.java index 37fe197280b..1d8329b4344 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetVersionDTO.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetVersionDTO.java @@ -47,6 +47,8 @@ public class DatasetVersionDTO { List fileMetadatas; List files; + String creationNote; + public boolean isInReview() { return inReview; } @@ -328,17 +330,17 @@ public List getDatasetFields() { return null; } + public String getCreationNote() { + return creationNote; + } + + public void setCreationNote(String creationNote) { + this.creationNote = creationNote; + } + @Override public String toString() { return "DatasetVersionDTO{" + "archiveNote=" + archiveNote + ", deacessionLink=" + deacessionLink + ", versionNumber=" + versionNumber + ", minorVersionNumber=" + versionMinorNumber + ", id=" + id + ", versionState=" + versionState + ", releaseDate=" + releaseDate + ", lastUpdateTime=" + lastUpdateTime + ", createTime=" + createTime + ", archiveTime=" + archiveTime + ", UNF=" + UNF + ", metadataBlocks=" + metadataBlocks + ", fileMetadatas=" + fileMetadatas + '}'; } - - - - - - - - - + } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index f5efc448090..ff775f8b44f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -377,6 +377,10 @@ private static void writeVersionStatement(XMLStreamWriter xmlw, DatasetVersionDT XmlWriterUtil.writeAttribute(xmlw,"type", datasetVersionDTO.getVersionState().toString()); xmlw.writeCharacters(datasetVersionDTO.getVersionNumber().toString()); xmlw.writeEndElement(); // version + xmlw.writeStartElement("notes"); + xmlw.writeCharacters(datasetVersionDTO.getCreationNote()); + xmlw.writeEndElement(); // version + xmlw.writeEndElement(); // verStmt } diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 8199b7d9c9f..0c4296845ba 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -1367,7 +1367,13 @@ private void writeDescriptions(XMLStreamWriter xmlw, DvObject dvObject, boolean } } - + String creationNote = dv.getCreationNote(); + if(!StringUtils.isBlank(creationNote)) { + attributes.clear(); + attributes.put("descriptionType", "TechnicalInfo"); + descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, creationNote); + } } if (descriptionsWritten) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 1bdee48b14d..6bd6d90d1c0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -445,7 +445,8 @@ public static JsonObjectBuilder json(DatasetVersion dsv, List anonymized .add("createTime", format(dsv.getCreateTime())) .add("alternativePersistentId", dataset.getAlternativePersistentIdentifier()) .add("publicationDate", dataset.getPublicationDateFormattedYYYYMMDD()) - .add("citationDate", dataset.getCitationDateFormattedYYYYMMDD()); + .add("citationDate", dataset.getCitationDateFormattedYYYYMMDD()) + .add("creationNote", dsv.getCreationNote()); License license = DatasetUtil.getLicense(dsv); if (license != null) { From 49a191a6c612d281a8657ee2f1ae24ece8891fc5 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 1 Nov 2024 15:31:33 -0400 Subject: [PATCH 0153/1048] add null check --- .../harvard/iq/dataverse/export/ddi/DdiExportUtil.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index ff775f8b44f..98e7bf58086 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -377,9 +377,11 @@ private static void writeVersionStatement(XMLStreamWriter xmlw, DatasetVersionDT XmlWriterUtil.writeAttribute(xmlw,"type", datasetVersionDTO.getVersionState().toString()); xmlw.writeCharacters(datasetVersionDTO.getVersionNumber().toString()); xmlw.writeEndElement(); // version - xmlw.writeStartElement("notes"); - xmlw.writeCharacters(datasetVersionDTO.getCreationNote()); - xmlw.writeEndElement(); // version + if (!StringUtils.isBlank(datasetVersionDTO.getCreationNote())) { + xmlw.writeStartElement("notes"); + xmlw.writeCharacters(datasetVersionDTO.getCreationNote()); + xmlw.writeEndElement(); // notes + } xmlw.writeEndElement(); // verStmt } From 95e71421f98dee43e655e9759feb13fbe5a89807 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 1 Nov 2024 15:36:42 -0400 Subject: [PATCH 0154/1048] fix curate command handling of creation note --- .../command/impl/CuratePublishedDatasetVersionCommand.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index e6e8279a314..d172fb728e4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -72,6 +72,10 @@ public Dataset execute(CommandContext ctxt) throws CommandException { TermsOfUseAndAccess newTerms = newVersion.getTermsOfUseAndAccess(); newTerms.setDatasetVersion(updateVersion); updateVersion.setTermsOfUseAndAccess(newTerms); + + //Creation Note + updateVersion.setCreationNote(newVersion.getCreationNote()); + // Clear unnecessary terms relationships .... newVersion.setTermsOfUseAndAccess(null); oldTerms.setDatasetVersion(null); From 76e13f63946ff1d5c837cd5e5837e9be37c688ca Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 1 Nov 2024 16:21:34 -0400 Subject: [PATCH 0155/1048] fix update --- src/main/webapp/dataset.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index c60dda14104..9ac54b13416 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -2003,7 +2003,7 @@
    + PF('blockDatasetForm').hide();" action="#{DatasetPage.saveCreationNote()}" update=":datasetForm,:messagePanel"/>
    - - + + + + + +
    Link From 82c44c2a59b87f6f220eec2440de498261d11952 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 13 Nov 2024 12:47:50 -0500 Subject: [PATCH 0168/1048] add sorted styles list, start retriever style method --- .../harvard/iq/dataverse/util/CSLUtil.java | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 src/main/java/edu/harvard/iq/dataverse/util/CSLUtil.java diff --git a/src/main/java/edu/harvard/iq/dataverse/util/CSLUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/CSLUtil.java new file mode 100644 index 00000000000..a392373c7cd --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/CSLUtil.java @@ -0,0 +1,100 @@ +package edu.harvard.iq.dataverse.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.logging.Logger; + +import de.undercouch.citeproc.CSL; +import jakarta.ejb.Singleton; + +@Singleton +public class CSLUtil { + private static final Logger logger = Logger.getLogger(CSLUtil.class.getName()); + + ArrayList supportedStyles; + + public List getSupportedStyles() { + if (supportedStyles!= null) { + return supportedStyles; + } + supportedStyles = new ArrayList<>(); + try { + supportedStyles = new ArrayList<>(CSL.getSupportedStyles()); + } catch (IOException e) { + logger.warning("Unable to retrieve supported CSL styles: " + e.getMessage()); + e.printStackTrace(); + } + supportedStyles.sort(Comparator.naturalOrder()); + return supportedStyles; + } + + /** + * Adapted from private retrieveStyle method in de.undercouch.citeproc.CSL + * Retrieves a CSL style from the classpath. For example, if the given name + * is ieee this method will load the file /ieee.csl + * @param styleName the style's name + * @return the serialized XML representation of the style + * @throws IOException if the style could not be loaded + */ + public String getCitationFormat(String citationKey) { + /** + * Retrieves a CSL style from the classpath. For example, if the given name + * is ieee this method will load the file /ieee.csl + * @param styleName the style's name + * @return the serialized XML representation of the style + * @throws IOException if the style could not be loaded + */ + private static String retrieveStyle(String styleName) throws IOException { + URL url; + if (styleName.startsWith("http://") || styleName.startsWith("https://")) { + try { + // try to load matching style from classpath + return retrieveStyle(styleName.substring(styleName.lastIndexOf('/') + 1)); + } catch (FileNotFoundException e) { + // there is no matching style in classpath + url = new URL(styleName); + } + } else { + // normalize file name + if (!styleName.endsWith(".csl")) { + styleName = styleName + ".csl"; + } + if (!styleName.startsWith("/")) { + styleName = "/" + styleName; + } + + // try to find style in classpath + url = CSL.class.getResource(styleName); + if (url == null) { + throw new FileNotFoundException("Could not find style in " + + "classpath: " + styleName); + } + } + + // load style + String result = CSLUtils.readURLToString(url, "UTF-8"); + + // handle dependent styles + if (isDependent(result)) { + String independentParentLink; + try { + independentParentLink = getIndependentParentLink(result); + } catch (ParserConfigurationException | IOException | SAXException e) { + throw new IOException("Could not load independent parent style", e); + } + if (independentParentLink == null) { + throw new IOException("Dependent style does not have an " + + "independent parent"); + } + return retrieveStyle(independentParentLink); + } + + return result; + } + } + + } + +} From 9a62528e704b65878b6b309e5400f6d0c1a93848 Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 15 Nov 2024 12:06:20 +0000 Subject: [PATCH 0169/1048] Added: API_BEARER_AUTH_JSON_CLAIMS feature flag --- .../harvard/iq/dataverse/settings/FeatureFlags.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java index 20632c170e4..5c9e1d6279c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java @@ -36,6 +36,18 @@ public enum FeatureFlags { * @since Dataverse @TODO: */ API_BEARER_AUTH("api-bearer-auth"), + /** + * Enables sending the missing user claims from the JSON provided during OIDC user registration + * (see API endpoint /users/register) when these claims are not returned by the identity provider + * but are necessary for registering the IdP user in Dataverse. + * + *

    The value of this feature flag is only considered when the feature flag + * {@link #API_BEARER_AUTH} is enabled.

    + * + * @apiNote Raise flag by setting "dataverse.feature.api-bearer-auth-json-claims" + * @since Dataverse @TODO: + */ + API_BEARER_AUTH_JSON_CLAIMS("api-bearer-auth-json-claims"), /** * For published (public) objects, don't use a join when searching Solr. * Experimental! Requires a reindex with the following feature flag enabled, From 0f2cfdcfe50ea9caca7b47023b70b7aff2a562ca Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 15 Nov 2024 12:38:52 +0000 Subject: [PATCH 0170/1048] Changed: renamed flag API_BEARER_AUTH_PROVIDE_MISSING_CLAIMS --- .../edu/harvard/iq/dataverse/settings/FeatureFlags.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java index 5c9e1d6279c..42f37034d90 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java @@ -37,17 +37,17 @@ public enum FeatureFlags { */ API_BEARER_AUTH("api-bearer-auth"), /** - * Enables sending the missing user claims from the JSON provided during OIDC user registration + * Enables sending the missing user claims in the request JSON provided during OIDC user registration * (see API endpoint /users/register) when these claims are not returned by the identity provider - * but are necessary for registering the IdP user in Dataverse. + * but are necessary for registering the user in Dataverse. * *

    The value of this feature flag is only considered when the feature flag * {@link #API_BEARER_AUTH} is enabled.

    * - * @apiNote Raise flag by setting "dataverse.feature.api-bearer-auth-json-claims" + * @apiNote Raise flag by setting "dataverse.feature.api-bearer-auth-provide-missing-claims" * @since Dataverse @TODO: */ - API_BEARER_AUTH_JSON_CLAIMS("api-bearer-auth-json-claims"), + API_BEARER_AUTH_PROVIDE_MISSING_CLAIMS("api-bearer-auth-provide-missing-claims"), /** * For published (public) objects, don't use a join when searching Solr. * Experimental! Requires a reindex with the following feature flag enabled, From 52a5a9e8d59ed041814269bb16a7f3fad990472c Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 15 Nov 2024 12:39:51 +0000 Subject: [PATCH 0171/1048] Added: API_BEARER_AUTH_PROVIDE_MISSING_CLAIMS management an different logic paths depending on the value to RegisterOIDCUserCommand --- .../command/impl/RegisterOIDCUserCommand.java | 77 ++++++++++--------- src/main/java/propertyFiles/Bundle.properties | 12 ++- 2 files changed, 47 insertions(+), 42 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommand.java index a82e6b57b68..57bf7832f62 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommand.java @@ -12,6 +12,7 @@ import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; import edu.harvard.iq.dataverse.engine.command.exception.InvalidFieldsCommandException; +import edu.harvard.iq.dataverse.settings.FeatureFlags; import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.HashMap; @@ -40,12 +41,9 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { } UserInfo userClaimsInfo = oidcUserInfo.getUserClaimsInfo(); + boolean provideMissingClaimsEnabled = FeatureFlags.API_BEARER_AUTH_PROVIDE_MISSING_CLAIMS.enabled(); - // Update the UserDTO object with available OIDC user claims; keep existing values if claims are absent - userDTO.setUsername(getValueOrDefault(userClaimsInfo.getPreferredUsername(), userDTO.getUsername())); - userDTO.setFirstName(getValueOrDefault(userClaimsInfo.getGivenName(), userDTO.getFirstName())); - userDTO.setLastName(getValueOrDefault(userClaimsInfo.getFamilyName(), userDTO.getLastName())); - userDTO.setEmailAddress(getValueOrDefault(userClaimsInfo.getEmailAddress(), userDTO.getEmailAddress())); + updateUserDTO(userClaimsInfo, provideMissingClaimsEnabled); AuthenticatedUserDisplayInfo userDisplayInfo = new AuthenticatedUserDisplayInfo( userDTO.getFirstName(), @@ -55,7 +53,7 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { userDTO.getPosition() != null ? userDTO.getPosition() : "" ); - Map fieldErrors = validateUserFields(ctxt); + Map fieldErrors = validateUserFields(ctxt, provideMissingClaimsEnabled); if (!fieldErrors.isEmpty()) { throw new InvalidFieldsCommandException( BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.invalidFields"), @@ -71,19 +69,34 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { } } + private void updateUserDTO(UserInfo userClaimsInfo, boolean provideMissingClaimsEnabled) { + if (provideMissingClaimsEnabled) { + // Update with available OIDC claims, keep existing values if claims are absent + userDTO.setUsername(getValueOrDefault(userClaimsInfo.getPreferredUsername(), userDTO.getUsername())); + userDTO.setFirstName(getValueOrDefault(userClaimsInfo.getGivenName(), userDTO.getFirstName())); + userDTO.setLastName(getValueOrDefault(userClaimsInfo.getFamilyName(), userDTO.getLastName())); + userDTO.setEmailAddress(getValueOrDefault(userClaimsInfo.getEmailAddress(), userDTO.getEmailAddress())); + } else { + // Always use the claims from the IdP provider + userDTO.setUsername(userClaimsInfo.getPreferredUsername()); + userDTO.setFirstName(userClaimsInfo.getGivenName()); + userDTO.setLastName(userClaimsInfo.getFamilyName()); + userDTO.setEmailAddress(userClaimsInfo.getEmailAddress()); + } + } + private String getValueOrDefault(String oidcValue, String dtoValue) { return (oidcValue == null || oidcValue.isEmpty()) ? dtoValue : oidcValue; } - private Map validateUserFields(CommandContext ctxt) { + private Map validateUserFields(CommandContext ctxt, boolean provideMissingClaimsEnabled) { Map fieldErrors = new HashMap<>(); validateTermsAccepted(fieldErrors); - validateEmailAddress(ctxt, fieldErrors); - validateUsername(ctxt, fieldErrors); - - validateRequiredField("firstName", userDTO.getFirstName(), "registerOidcUserCommand.errors.firstNameFieldRequired", fieldErrors); - validateRequiredField("lastName", userDTO.getLastName(), "registerOidcUserCommand.errors.lastNameFieldRequired", fieldErrors); + validateField(fieldErrors, "emailAddress", userDTO.getEmailAddress(), ctxt, provideMissingClaimsEnabled); + validateField(fieldErrors, "username", userDTO.getUsername(), ctxt, provideMissingClaimsEnabled); + validateField(fieldErrors, "firstName", userDTO.getFirstName(), ctxt, provideMissingClaimsEnabled); + validateField(fieldErrors, "lastName", userDTO.getLastName(), ctxt, provideMissingClaimsEnabled); return fieldErrors; } @@ -94,35 +107,23 @@ private void validateTermsAccepted(Map fieldErrors) { } } - private void validateEmailAddress(CommandContext ctxt, Map fieldErrors) { - String emailAddress = userDTO.getEmailAddress(); - if (emailAddress == null || emailAddress.isEmpty()) { - fieldErrors.put("emailAddress", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.emailFieldRequired")); - } else if (isEmailInUse(ctxt, emailAddress)) { - fieldErrors.put("emailAddress", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.emailAddressInUse")); - } - } - - private void validateUsername(CommandContext ctxt, Map fieldErrors) { - String username = userDTO.getUsername(); - if (username == null || username.isEmpty()) { - fieldErrors.put("username", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.usernameFieldRequired")); - } else if (isUsernameInUse(ctxt, username)) { - fieldErrors.put("username", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.usernameInUse")); - } - } - - private void validateRequiredField(String fieldName, String fieldValue, String bundleKey, Map fieldErrors) { + private void validateField(Map fieldErrors, String fieldName, String fieldValue, CommandContext ctxt, boolean provideMissingClaimsEnabled) { if (fieldValue == null || fieldValue.isEmpty()) { - fieldErrors.put(fieldName, BundleUtil.getStringFromBundle(bundleKey)); + String errorKey = provideMissingClaimsEnabled ? + "registerOidcUserCommand.errors.provideMissingClaimsEnabled." + fieldName + "FieldRequired" : + "registerOidcUserCommand.errors.provideMissingClaimsDisabled." + fieldName + "FieldRequired"; + fieldErrors.put(fieldName, BundleUtil.getStringFromBundle(errorKey)); + } else if (isFieldInUse(ctxt, fieldName, fieldValue)) { + fieldErrors.put(fieldName, BundleUtil.getStringFromBundle("registerOidcUserCommand.errors." + fieldName + "InUse")); } } - private boolean isEmailInUse(CommandContext ctxt, String emailAddress) { - return ctxt.authentication().getAuthenticatedUserByEmail(emailAddress) != null; - } - - private boolean isUsernameInUse(CommandContext ctxt, String username) { - return ctxt.authentication().getAuthenticatedUser(username) != null; + private boolean isFieldInUse(CommandContext ctxt, String fieldName, String value) { + if ("emailAddress".equals(fieldName)) { + return ctxt.authentication().getAuthenticatedUserByEmail(value) != null; + } else if ("username".equals(fieldName)) { + return ctxt.authentication().getAuthenticatedUser(value) != null; + } + return false; } } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 9ea87440535..e2fc48054e6 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -3072,10 +3072,14 @@ users.api.userRegistered=User registered. registerOidcUserCommand.errors.userAlreadyRegisteredWithToken=User is already registered with this token. registerOidcUserCommand.errors.invalidFields=The provided fields are invalid for registering a new user. registerOidcUserCommand.errors.userShouldAcceptTerms=Terms should be accepted. -registerOidcUserCommand.errors.emailFieldRequired=It is required to include an emailAddress field in the request JSON for registering the user. -registerOidcUserCommand.errors.usernameFieldRequired=It is required to include a username field in the request JSON for registering the user. -registerOidcUserCommand.errors.firstNameFieldRequired=It is required to include a firstName field in the request JSON for registering the user. -registerOidcUserCommand.errors.lastNameFieldRequired=It is required to include a lastName field in the request JSON for registering the user. +registerOidcUserCommand.errors.provideMissingClaimsEnabled.emailAddressFieldRequired=It is required to include an emailAddress field in the request JSON for registering the user. +registerOidcUserCommand.errors.provideMissingClaimsDisabled.emailAddressFieldRequired=The OIDC identity provider does not provide the user claim 'email', which is required for user registration. Please contact your identity provider. +registerOidcUserCommand.errors.provideMissingClaimsEnabled.usernameFieldRequired=It is required to include a username field in the request JSON for registering the user. +registerOidcUserCommand.errors.provideMissingClaimsDisabled.usernameFieldRequired=The OIDC identity provider does not provide the user claim 'preferred_username', which is required for user registration. Please contact your identity provider. +registerOidcUserCommand.errors.provideMissingClaimsEnabled.firstNameFieldRequired=It is required to include a firstName field in the request JSON for registering the user. +registerOidcUserCommand.errors.provideMissingClaimsDisabled.firstNameFieldRequired=The OIDC identity provider does not provide the user claim 'given_name', which is required for user registration. Please contact your identity provider. +registerOidcUserCommand.errors.provideMissingClaimsEnabled.lastNameFieldRequired=It is required to include a lastName field in the request JSON for registering the user. +registerOidcUserCommand.errors.provideMissingClaimsDisabled.lastNameFieldRequired=The OIDC identity provider does not provide the user claim 'family_name', which is required for user registration. Please contact your identity provider. registerOidcUserCommand.errors.emailAddressInUse=Email already in use. registerOidcUserCommand.errors.usernameInUse=Username already in use. From 047a14c48e0a6a29cf844190ac43dd235e4b842b Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 15 Nov 2024 13:02:24 +0000 Subject: [PATCH 0172/1048] Fixed: RegisterOIDCUserCommandTest --- .../impl/RegisterOIDCUserCommandTest.java | 46 ++++++++++++++++--- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommandTest.java index bd9edf150f6..30fc7687c55 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommandTest.java @@ -13,7 +13,10 @@ import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.engine.command.exception.InvalidFieldsCommandException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.testing.JvmSetting; +import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.InjectMocks; @@ -25,6 +28,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.*; +@LocalJvmSettings class RegisterOIDCUserCommandTest { private static final String TEST_BEARER_TOKEN = "Bearer test"; @@ -69,7 +73,7 @@ private void setUpDefaultUserDTO() { } @Test - public void execute_unacceptedTerms_availableEmailAndUsername() throws AuthorizationException { + public void execute_completedUserDTOWithUnacceptedTerms_provideMissingClaimsDisabled() throws AuthorizationException { userDTO.setTermsAccepted(false); when(authServiceMock.getAuthenticatedUserByEmail(userDTO.getEmailAddress())).thenReturn(null); when(authServiceMock.getAuthenticatedUser(userDTO.getUsername())).thenReturn(null); @@ -81,13 +85,41 @@ public void execute_unacceptedTerms_availableEmailAndUsername() throws Authoriza InvalidFieldsCommandException ex = (InvalidFieldsCommandException) exception; assertThat(ex.getFieldErrors()) .containsEntry("termsAccepted", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.userShouldAcceptTerms")) - .doesNotContainEntry("emailAddress", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.emailAddressInUse")) - .doesNotContainEntry("username", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.usernameInUse")); + .containsEntry("emailAddress", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsDisabled.emailAddressFieldRequired")) + .containsEntry("username", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsDisabled.usernameFieldRequired")) + .containsEntry("firstName", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsDisabled.firstNameFieldRequired")) + .containsEntry("lastName", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsDisabled.lastNameFieldRequired")); }); } @Test - public void execute_acceptedTerms_availableEmailAndUsername() throws AuthorizationException { + @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth-provide-missing-claims") + public void execute_uncompletedUserDTOWithUnacceptedTerms_provideMissingClaimsEnabled() throws AuthorizationException { + userDTO.setTermsAccepted(false); + userDTO.setEmailAddress(null); + userDTO.setUsername(null); + userDTO.setFirstName(null); + userDTO.setLastName(null); + when(authServiceMock.getAuthenticatedUserByEmail(userDTO.getEmailAddress())).thenReturn(null); + when(authServiceMock.getAuthenticatedUser(userDTO.getUsername())).thenReturn(null); + when(authServiceMock.verifyOIDCBearerTokenAndGetUserIdentifier(TEST_BEARER_TOKEN)).thenReturn(OIDCUserInfoMock); + + assertThatThrownBy(() -> sut.execute(context)) + .isInstanceOf(InvalidFieldsCommandException.class) + .satisfies(exception -> { + InvalidFieldsCommandException ex = (InvalidFieldsCommandException) exception; + assertThat(ex.getFieldErrors()) + .containsEntry("termsAccepted", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.userShouldAcceptTerms")) + .containsEntry("emailAddress", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.emailAddressFieldRequired")) + .containsEntry("username", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.usernameFieldRequired")) + .containsEntry("firstName", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.firstNameFieldRequired")) + .containsEntry("lastName", BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.lastNameFieldRequired")); + }); + } + + @Test + @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth-provide-missing-claims") + public void execute_acceptedTerms_unavailableEmailAndUsername_provideMissingClaimsEnabled() throws AuthorizationException { when(authServiceMock.getAuthenticatedUserByEmail(userDTO.getEmailAddress())).thenReturn(existingTestUser); when(authServiceMock.getAuthenticatedUser(userDTO.getUsername())).thenReturn(existingTestUser); when(authServiceMock.verifyOIDCBearerTokenAndGetUserIdentifier(TEST_BEARER_TOKEN)).thenReturn(OIDCUserInfoMock); @@ -130,7 +162,8 @@ void execute_throwsIllegalCommandException_ifUserAlreadyRegisteredWithToken() th } @Test - void execute_happyPath_withoutAffiliationAndPosition() throws AuthorizationException, CommandException { + @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth-provide-missing-claims") + void execute_happyPath_withoutAffiliationAndPosition_provideMissingClaimsEnabled() throws AuthorizationException, CommandException { when(authServiceMock.verifyOIDCBearerTokenAndGetUserIdentifier(TEST_BEARER_TOKEN)).thenReturn(OIDCUserInfoMock); sut.execute(context); @@ -150,7 +183,8 @@ void execute_happyPath_withoutAffiliationAndPosition() throws AuthorizationExcep } @Test - void execute_happyPath_withAffiliationAndPosition() throws AuthorizationException, CommandException { + @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth-provide-missing-claims") + void execute_happyPath_withAffiliationAndPosition_provideMissingClaimsEnabled() throws AuthorizationException, CommandException { userDTO.setPosition("test position"); userDTO.setAffiliation("test affiliation"); From cc86a8307405344bcd1881a42d3d7a8f8ab26b5a Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 15 Nov 2024 13:14:48 +0000 Subject: [PATCH 0173/1048] Added: explanatory comment tweak --- .../dataverse/engine/command/impl/RegisterOIDCUserCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommand.java index 57bf7832f62..e580c1ad7cc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommand.java @@ -77,7 +77,7 @@ private void updateUserDTO(UserInfo userClaimsInfo, boolean provideMissingClaims userDTO.setLastName(getValueOrDefault(userClaimsInfo.getFamilyName(), userDTO.getLastName())); userDTO.setEmailAddress(getValueOrDefault(userClaimsInfo.getEmailAddress(), userDTO.getEmailAddress())); } else { - // Always use the claims from the IdP provider + // Always use the claims provided by the OIDC provider, regardless of whether they are null or not userDTO.setUsername(userClaimsInfo.getPreferredUsername()); userDTO.setFirstName(userClaimsInfo.getGivenName()); userDTO.setLastName(userClaimsInfo.getFamilyName()); From c3de7d735cdebcbbec69511928369683b9c7c5fa Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 15 Nov 2024 13:27:42 +0000 Subject: [PATCH 0174/1048] Added: DATAVERSE_FEATURE_API_BEARER_AUTH_PROVIDE_MISSING_CLAIMS enabled in docker-compose-dev --- docker-compose-dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 384b70b7a7b..3f5cae1b263 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -17,6 +17,7 @@ services: SKIP_DEPLOY: "${SKIP_DEPLOY}" DATAVERSE_JSF_REFRESH_PERIOD: "1" DATAVERSE_FEATURE_API_BEARER_AUTH: "1" + DATAVERSE_FEATURE_API_BEARER_AUTH_PROVIDE_MISSING_CLAIMS: "1" DATAVERSE_MAIL_SYSTEM_EMAIL: "dataverse@localhost" DATAVERSE_MAIL_MTA_HOST: "smtp" DATAVERSE_AUTH_OIDC_ENABLED: "1" From 25cdf98d2cba0dc672161f8cafa8d363c65c8c10 Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 15 Nov 2024 13:28:00 +0000 Subject: [PATCH 0175/1048] Fixed: UsersIT registerOidcUser --- src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java index acd5bd658e0..cb4a2b862c9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java @@ -627,9 +627,9 @@ public void testRegisterOIDCUser() { registerOidcUserResponse.then().assertThat() .statusCode(BAD_REQUEST.getStatusCode()) .body("message", equalTo(BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.invalidFields"))) - .body("fieldErrors.firstName", equalTo(BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.firstNameFieldRequired"))) - .body("fieldErrors.lastName", equalTo(BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.lastNameFieldRequired"))) - .body("fieldErrors.emailAddress", equalTo(BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.emailFieldRequired"))); + .body("fieldErrors.firstName", equalTo(BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.firstNameFieldRequired"))) + .body("fieldErrors.lastName", equalTo(BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.lastNameFieldRequired"))) + .body("fieldErrors.emailAddress", equalTo(BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.provideMissingClaimsEnabled.emailAddressFieldRequired"))); // Should register user when the Bearer token is valid and the provided User JSON contains the missing claims in the IdP registerOidcUserResponse = UtilIT.registerOidcUser( From 5d39ac187e3823e0b6ac914c7c341558e9da5d75 Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 15 Nov 2024 13:56:22 +0000 Subject: [PATCH 0176/1048] Added: #10959 docs to auth.rst --- doc/sphinx-guides/source/api/auth.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/doc/sphinx-guides/source/api/auth.rst b/doc/sphinx-guides/source/api/auth.rst index eae3bd3c969..d30d0097802 100644 --- a/doc/sphinx-guides/source/api/auth.rst +++ b/doc/sphinx-guides/source/api/auth.rst @@ -81,6 +81,29 @@ To test if bearer tokens are working, you can try something like the following ( curl -H "Authorization: Bearer $TOKEN" http://localhost:8080/api/users/:me +It may happen that when you try to authenticate a user for the first time with a bearer token, it does not have an associated user account in Dataverse. In this case, it is necessary to register the user using the following endpoint: + +.. code-block:: bash + + curl -H "Authorization: Bearer $TOKEN" -X POST http://localhost:8080/api/users/register --data '{"termsAccepted":true}' + +It is essential to send a JSON that includes the property ``termsAccepted`` set to true, which indicates that you accept the terms of service of Dataverse. Otherwise, you will not be able to create an account. + +In this JSON, we can also include the fields ``position`` or ``affiliation``, in the same way as when we register a user through the Dataverse UI. These fields are optional, and if not provided, they will be persisted as empty in Dataverse. + +Beyond the ``api-bearer-auth`` feature flag, there is another flag called ``api-bearer-auth-json-claims`` that can be enabled to allow sending missing user claims in the registration JSON. This is useful when the identity provider does not supply the necessary claims. However, this flag will only be considered if the ``api-bearer-auth`` feature flag is enabled. If the latter is not enabled, the ``api-bearer-auth-json-claims`` flag will be ignored. + +With the ``api-bearer-auth`` feature flag enabled, you can include the following properties in the request JSON: + +- ``username`` +- ``firstName`` +- ``lastName`` +- ``emailAddress`` + +Note that even if they are included in the JSON, if it is possible to retrieve the corresponding claims from the identity provider, these values will be ignored and the ones from the IdP will be used instead. + +This functionality is included under a feature flag because using it may introduce potential security risks, such as user impersonation, if the identity provider does not provide an email field and the user submits an email address they do not own. + Signed URLs ----------- From a438c8a8bc6d7ea1762cb3b117d781478f4d8959 Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 15 Nov 2024 14:03:28 +0000 Subject: [PATCH 0177/1048] Added: docs for #10959 --- doc/sphinx-guides/source/api/auth.rst | 2 +- doc/sphinx-guides/source/installation/config.rst | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/auth.rst b/doc/sphinx-guides/source/api/auth.rst index d30d0097802..101e283d5b1 100644 --- a/doc/sphinx-guides/source/api/auth.rst +++ b/doc/sphinx-guides/source/api/auth.rst @@ -102,7 +102,7 @@ With the ``api-bearer-auth`` feature flag enabled, you can include the following Note that even if they are included in the JSON, if it is possible to retrieve the corresponding claims from the identity provider, these values will be ignored and the ones from the IdP will be used instead. -This functionality is included under a feature flag because using it may introduce potential security risks, such as user impersonation, if the identity provider does not provide an email field and the user submits an email address they do not own. +This functionality is included under a feature flag because using it may introduce user impersonation issues, for example if the identity provider does not provide an email field and the user submits an email address they do not own. Signed URLs ----------- diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index e3965e3cd7c..f7ccf7e1698 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3343,6 +3343,12 @@ please find all known feature flags below. Any of these flags can be activated u * - api-session-auth - Enables API authentication via session cookie (JSESSIONID). **Caution: Enabling this feature flag exposes the installation to CSRF risks!** We expect this feature flag to be temporary (only used by frontend developers, see `#9063 `_) and for the feature to be removed in the future. - ``Off`` + * - api-bearer-auth + - Enables API authentication via Bearer Token. + - ``Off`` + * - api-bearer-auth-provide-missing-claims + - Enables sending missing user claims in the request JSON provided during OIDC user registration, when these claims are not returned by the identity provider and are required for registration. This feature only works when the feature flag ``api-bearer-auth`` is also enabled. **Caution: Enabling this feature flag exposes the installation to potential user impersonation issues.** + - ``Off`` * - avoid-expensive-solr-join - Changes the way Solr queries are constructed for public content (published Collections, Datasets and Files). It removes a very expensive Solr join on all such documents, improving overall performance, especially for large instances under heavy load. Before this feature flag is enabled, the corresponding indexing feature (see next feature flag) must be turned on and a full reindex performed (otherwise public objects are not going to be shown in search results). See :doc:`/admin/solr-search-index`. - ``Off`` From 9ba377ee05e0cc671eee14900de84cc15f50a431 Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 15 Nov 2024 14:05:37 +0000 Subject: [PATCH 0178/1048] Fixed: doc tweak --- doc/sphinx-guides/source/api/auth.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/auth.rst b/doc/sphinx-guides/source/api/auth.rst index 101e283d5b1..ca68e507b9b 100644 --- a/doc/sphinx-guides/source/api/auth.rst +++ b/doc/sphinx-guides/source/api/auth.rst @@ -100,7 +100,7 @@ With the ``api-bearer-auth`` feature flag enabled, you can include the following - ``lastName`` - ``emailAddress`` -Note that even if they are included in the JSON, if it is possible to retrieve the corresponding claims from the identity provider, these values will be ignored and the ones from the IdP will be used instead. +Note that even if they are included in the JSON, if it is possible to retrieve the corresponding claims from the identity provider, these values will be ignored and the ones from the identity provider will be used instead. This functionality is included under a feature flag because using it may introduce user impersonation issues, for example if the identity provider does not provide an email field and the user submits an email address they do not own. From b00ac7f3f76d3f5564186b68a2255fac5a4156a4 Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 15 Nov 2024 14:09:10 +0000 Subject: [PATCH 0179/1048] Changed: replaced version TODO with 5.14 for api-bearer-auth feature flag doc --- .../java/edu/harvard/iq/dataverse/settings/FeatureFlags.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java index 42f37034d90..b3774c3fe06 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java @@ -33,7 +33,7 @@ public enum FeatureFlags { /** * Enables API authentication via Bearer Token. * @apiNote Raise flag by setting "dataverse.feature.api-bearer-auth" - * @since Dataverse @TODO: + * @since Dataverse 5.14: */ API_BEARER_AUTH("api-bearer-auth"), /** From 7921f0b5a0908c8c7b4af310e23294641549573e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 15 Nov 2024 11:48:17 -0500 Subject: [PATCH 0180/1048] working version --- .../harvard/iq/dataverse/util/CSLUtil.java | 129 +++++++----------- src/main/webapp/dataset.xhtml | 75 +++++++--- 2 files changed, 103 insertions(+), 101 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/CSLUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/CSLUtil.java index a392373c7cd..c01be52dc0e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/CSLUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/CSLUtil.java @@ -1,100 +1,71 @@ package edu.harvard.iq.dataverse.util; +import java.io.FileNotFoundException; import java.io.IOException; +import java.net.URL; import java.util.ArrayList; import java.util.Comparator; import java.util.List; +import java.util.Set; import java.util.logging.Logger; import de.undercouch.citeproc.CSL; +import de.undercouch.citeproc.helper.CSLUtils; import jakarta.ejb.Singleton; -@Singleton public class CSLUtil { private static final Logger logger = Logger.getLogger(CSLUtil.class.getName()); - ArrayList supportedStyles; - - public List getSupportedStyles() { - if (supportedStyles!= null) { - return supportedStyles; - } - supportedStyles = new ArrayList<>(); - try { - supportedStyles = new ArrayList<>(CSL.getSupportedStyles()); - } catch (IOException e) { - logger.warning("Unable to retrieve supported CSL styles: " + e.getMessage()); - e.printStackTrace(); - } - supportedStyles.sort(Comparator.naturalOrder()); + static ArrayList supportedStyles; + + public static List getSupportedStyles() { + if (supportedStyles != null) { return supportedStyles; } - - /** - * Adapted from private retrieveStyle method in de.undercouch.citeproc.CSL - * Retrieves a CSL style from the classpath. For example, if the given name - * is ieee this method will load the file /ieee.csl - * @param styleName the style's name - * @return the serialized XML representation of the style - * @throws IOException if the style could not be loaded - */ - public String getCitationFormat(String citationKey) { - /** - * Retrieves a CSL style from the classpath. For example, if the given name - * is ieee this method will load the file /ieee.csl - * @param styleName the style's name - * @return the serialized XML representation of the style - * @throws IOException if the style could not be loaded - */ - private static String retrieveStyle(String styleName) throws IOException { - URL url; - if (styleName.startsWith("http://") || styleName.startsWith("https://")) { - try { - // try to load matching style from classpath - return retrieveStyle(styleName.substring(styleName.lastIndexOf('/') + 1)); - } catch (FileNotFoundException e) { - // there is no matching style in classpath - url = new URL(styleName); - } - } else { - // normalize file name - if (!styleName.endsWith(".csl")) { - styleName = styleName + ".csl"; - } - if (!styleName.startsWith("/")) { - styleName = "/" + styleName; - } - - // try to find style in classpath - url = CSL.class.getResource(styleName); - if (url == null) { - throw new FileNotFoundException("Could not find style in " - + "classpath: " + styleName); - } - } - - // load style - String result = CSLUtils.readURLToString(url, "UTF-8"); + supportedStyles = new ArrayList<>(); + try { + Set styleSet = CSL.getSupportedStyles(); + // Remove styles starting with "dependent/" + styleSet.removeIf(style -> style.startsWith("dependent/")); + supportedStyles = new ArrayList<>(styleSet); + } catch (IOException e) { + logger.warning("Unable to retrieve supported CSL styles: " + e.getMessage()); + e.printStackTrace(); + } + supportedStyles.sort(Comparator.naturalOrder()); + return supportedStyles; + } - // handle dependent styles - if (isDependent(result)) { - String independentParentLink; - try { - independentParentLink = getIndependentParentLink(result); - } catch (ParserConfigurationException | IOException | SAXException e) { - throw new IOException("Could not load independent parent style", e); - } - if (independentParentLink == null) { - throw new IOException("Dependent style does not have an " - + "independent parent"); - } - return retrieveStyle(independentParentLink); - } + /** + * Adapted from private retrieveStyle method in de.undercouch.citeproc.CSL + * Retrieves a CSL style from the classpath. For example, if the given name is + * ieee this method will load the file /ieee.csl + * + * @param styleName the style's name + * @return the serialized XML representation of the style + * @throws IOException if the style could not be loaded + */ + public static String getCitationFormat(String styleName) throws IOException { + URL url; - return result; - } + // normalize file name + if (!styleName.endsWith(".csl")) { + styleName = styleName + ".csl"; } - + if (!styleName.startsWith("/")) { + styleName = "/" + styleName; } - + + // try to find style in classpath + url = CSL.class.getResource(styleName); + if (url == null) { + throw new FileNotFoundException("Could not find style in " + "classpath: " + styleName); + } + + // load style + String result = CSLUtils.readURLToString(url, "UTF-8"); + result=result.replace("\"", "\\\"").replace("\r","").replace("\n",""); + return result; + } + } diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 5075f365ffb..be221fd532f 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1971,28 +1971,56 @@
    - - - - - - - - -
    - Link -
    -
    - -
    -     - - + + + + + + + + + + + + Link + + + +
    + +
    +     + + @@ -2069,6 +2097,9 @@ function updateTemplate() { $('button[id$="updateTemplate"]').trigger('click'); } + function updateCSLCitation() { + $('button[id$="updateCSL"]').trigger('click'); + } function updateOwnerDataverse() { $('button[id$="updateOwnerDataverse"]').trigger('click'); } From 3adb19419792d9304a51e75e76619305b568aa3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Fri, 15 Nov 2024 17:51:36 +0100 Subject: [PATCH 0181/1048] revert feature-flag --- .../search/SolrClientIndexService.java | 17 ++++++----------- .../iq/dataverse/search/SolrClientService.java | 14 ++++---------- .../iq/dataverse/settings/FeatureFlags.java | 7 ------- .../search/SolrClientIndexServiceTest.java | 4 +--- .../dataverse/search/SolrClientServiceTest.java | 6 ++---- 5 files changed, 13 insertions(+), 35 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java index 59281d5fa4e..0b7f1aae798 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java @@ -1,16 +1,15 @@ package edu.harvard.iq.dataverse.search; +import java.util.logging.Logger; + import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient; import org.apache.solr.client.solrj.impl.Http2SolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import edu.harvard.iq.dataverse.settings.FeatureFlags; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import jakarta.ejb.Singleton; import jakarta.inject.Named; -import java.util.logging.Logger; /** * Solr client to provide insert/update/delete operations. @@ -19,20 +18,15 @@ @Named @Singleton public class SolrClientIndexService extends AbstractSolrClientService { + private static final Logger logger = Logger.getLogger(SolrClientIndexService.class.getCanonicalName()); private SolrClient solrClient; @PostConstruct public void init() { - if (FeatureFlags.ENABLE_HTTP2_SOLR_CLIENT.enabled()) { - solrClient = new ConcurrentUpdateHttp2SolrClient.Builder( - getSolrUrl(), new Http2SolrClient.Builder().build()).build(); - } else { - // ConcurrentUpdateSolrClient seem to be more suitable, but - // actually only HttpSolrClient is used. - solrClient = new HttpSolrClient.Builder(getSolrUrl()).build(); - } + solrClient = new ConcurrentUpdateHttp2SolrClient.Builder( + getSolrUrl(), new Http2SolrClient.Builder().build()).build(); } @PreDestroy @@ -51,4 +45,5 @@ public SolrClient getSolrClient() { public void setSolrClient(SolrClient solrClient) { this.solrClient = solrClient; } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java index 83f16e29af2..f9d94b8c6d3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java @@ -2,9 +2,7 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.Http2SolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import edu.harvard.iq.dataverse.settings.FeatureFlags; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import jakarta.ejb.Singleton; @@ -15,9 +13,9 @@ * * @author landreev * - * This singleton is dedicated to initializing the HttpSolrClient, or the Http2SolrClient - * (if feature-flag is enabled), used by the application to talk to the search engine, - * and serving it to all the other classes that need it. + * This singleton is dedicated to initializing the Http2SolrClient, used by + * the application to talk to the search engine, and serving it to all the + * other classes that need it. * This ensures that we are using one client only - as recommended by the * documentation. */ @@ -30,11 +28,7 @@ public class SolrClientService extends AbstractSolrClientService { @PostConstruct public void init() { - if (FeatureFlags.ENABLE_HTTP2_SOLR_CLIENT.enabled()) { - solrClient = new Http2SolrClient.Builder(getSolrUrl()).build(); - } else { - solrClient = new HttpSolrClient.Builder(getSolrUrl()).build(); - } + solrClient = new Http2SolrClient.Builder(getSolrUrl()).build(); } @PreDestroy diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java index 97a8eae1d9c..24b41c623b6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java @@ -67,13 +67,6 @@ public enum FeatureFlags { * @since Dataverse 6.3 */ INDEX_HARVESTED_METADATA_SOURCE("index-harvested-metadata-source"), - /** - * With this flag enabled, a the new Solr client Http2SolrClient is used in - * order to replace HttpSolrClient witch is deprecated since Solr 9. - * - * @apiNote Raise flag by setting "dataverse.feature.enable-http2-solr-client" - */ - ENABLE_HTTP2_SOLR_CLIENT("enable-http2-solr-client"), /** * Dataverse normally deletes all solr documents related to a dataset's files * when the dataset is reindexed. With this flag enabled, additional logic is diff --git a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java index 09d9e096e55..d3d68fa5f6a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java @@ -8,7 +8,6 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -49,7 +48,7 @@ void testInitWithDefaults() { // then SolrClient client = clientService.getSolrClient(); assertNotNull(client); - assertInstanceOf(HttpSolrClient.class, client); + assertInstanceOf(ConcurrentUpdateHttp2SolrClient.class, client); assertEquals(url, clientService.getSolrUrl()); } @@ -57,7 +56,6 @@ void testInitWithDefaults() { @JvmSetting(key = JvmSettings.SOLR_HOST, value = "foobar") @JvmSetting(key = JvmSettings.SOLR_PORT, value = "1234") @JvmSetting(key = JvmSettings.SOLR_CORE, value = "test") - @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "on", varArgs = "enable-http2-solr-client") void testInitWithConfig() { // given String url = "http://foobar:1234/solr/test"; diff --git a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java index 351d553f203..13cea4151ff 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java @@ -8,7 +8,6 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.Http2SolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -47,15 +46,14 @@ void testInitWithDefaults() { // then SolrClient client = clientService.getSolrClient(); assertNotNull(client); - assertInstanceOf(HttpSolrClient.class, client); - assertEquals(url, ((HttpSolrClient) client).getBaseURL()); + assertInstanceOf(Http2SolrClient.class, client); + assertEquals(url, clientService.getSolrUrl()); } @Test @JvmSetting(key = JvmSettings.SOLR_HOST, value = "foobar") @JvmSetting(key = JvmSettings.SOLR_PORT, value = "1234") @JvmSetting(key = JvmSettings.SOLR_CORE, value = "test") - @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "on", varArgs = "enable-http2-solr-client") void testInitWithConfig() { // given String url = "http://foobar:1234/solr/test"; From 6a05ecdcbbccae3fcc54711ba5a7b76dbd44049d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Fri, 15 Nov 2024 18:05:42 +0100 Subject: [PATCH 0182/1048] revert feature flag documentation --- doc/release-notes/10241-new-solr-client.md | 6 +++--- doc/sphinx-guides/source/installation/config.rst | 3 --- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/doc/release-notes/10241-new-solr-client.md b/doc/release-notes/10241-new-solr-client.md index 211ad9d1ec9..67ccdd4f184 100644 --- a/doc/release-notes/10241-new-solr-client.md +++ b/doc/release-notes/10241-new-solr-client.md @@ -1,9 +1,9 @@ -[HttpSolrClient](https://solr.apache.org/docs/9_3_0/solrj/org/apache/solr/client/solrj/impl/HttpSolrClient.html) is deprecated as of Solr 9, and which will be removed in a future major release of Solr. It's recommended to use [Http2SolrClient](https://solr.apache.org/docs/9_3_0/solrj/org/apache/solr/client/solrj/impl/Http2SolrClient.html) instead. +[HttpSolrClient](https://solr.apache.org/docs/9_4_1/solrj/org/apache/solr/client/solrj/impl/HttpSolrClient.html) is deprecated as of Solr 9, and which will be removed in a future major release of Solr. It's recommended to use [Http2SolrClient](https://solr.apache.org/docs/9_4_1/solrj/org/apache/solr/client/solrj/impl/Http2SolrClient.html) instead. [Solr documentation](https://solr.apache.org/guide/solr/latest/deployment-guide/solrj.html#types-of-solrclients) describe it as a _async, non-blocking and general-purpose client that leverage HTTP/2 using the Jetty Http library_. -With Solr 9.3.0, the Http2SolrClient is indicate as experimental. But since the 9.6 version of Solr, this mention is no longer maintained. +With Solr 9.4.1, the Http2SolrClient is indicate as experimental. But since the 9.6 version of Solr, this mention is no longer maintained. -For the time being, its activation is therefore conditional on the use of a [feature-flag](https://dataverse-guide--10241.org.readthedocs.build/en/10241/installation/config.html#feature-flags). Activation also enables ConcurrentUpdateHttp2SolrClient, which is supposed to be more efficient for indexing. +The ConcurrentUpdateHttp2SolrClient is now also used in some cases, which is supposed to be more efficient for indexing. For more information, see issue [#10161](https://github.com/IQSS/dataverse/issues/10161) and pull request [#10241](https://github.com/IQSS/dataverse/pull/10241) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 9823e3b6432..b4705aa6521 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3345,9 +3345,6 @@ please find all known feature flags below. Any of these flags can be activated u * - add-publicobject-solr-field - Adds an extra boolean field `PublicObject_b:true` for public content (published Collections, Datasets and Files). Once reindexed with these fields, we can rely on it to remove a very expensive Solr join on all such documents in Solr queries, significantly improving overall performance (by enabling the feature flag above, `avoid-expensive-solr-join`). These two flags are separate so that an instance can reindex their holdings before enabling the optimization in searches, thus avoiding having their public objects temporarily disappear from search results while the reindexing is in progress. - ``Off`` - * - enable-http2-solr-client - - Enable a new Solr client ``Http2SolrClient`` instead of ``HttpSolrClient`` witch is deprecated since Solr 9. Also enables use of ``ConcurrentUpdateHttp2SolrClient``, recommended to send concurrent updates to Solr. More informations about this Solr clients on `Solr documentation `_. - - ``Off`` * - reduce-solr-deletes - Avoids deleting and recreating solr documents for dataset files when reindexing. - ``Off`` From b0e76d25857f646889dd9a11de6727eee41c3eb5 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 15 Nov 2024 13:03:10 -0500 Subject: [PATCH 0183/1048] Styling, cleanup, ally --- src/main/java/propertyFiles/Bundle.properties | 6 ++ src/main/webapp/dataset-citation.xhtml | 4 +- src/main/webapp/dataset.xhtml | 98 +++++++++---------- src/main/webapp/resources/css/structure.css | 8 ++ 4 files changed, 64 insertions(+), 52 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 5f3e4c33e0b..5d4ec630950 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1606,6 +1606,12 @@ dataset.cite.downloadBtn=Cite Dataset dataset.cite.downloadBtn.xml=EndNote XML dataset.cite.downloadBtn.ris=RIS dataset.cite.downloadBtn.bib=BibTeX +dataset.cite.viewCitation=View Styled Citation +dataset.cite.cslDialog.title=Styled Citation +dataset.cite.cslDialog.select=Select a CSL style +dataset.cite.cslDialog.citation=Citation in {0} style +dataset.cite.cslDialog.generating=Generating Citation... +dataset.cite.cslDialog.close=Close dataset.create.authenticatedUsersOnly=Only authenticated users can create datasets. dataset.deaccession.reason=Deaccession Reason dataset.beAccessedAt=The dataset can now be accessed at: diff --git a/src/main/webapp/dataset-citation.xhtml b/src/main/webapp/dataset-citation.xhtml index 3ec301d71bc..26bc9dca65e 100644 --- a/src/main/webapp/dataset-citation.xhtml +++ b/src/main/webapp/dataset-citation.xhtml @@ -42,8 +42,8 @@
  • -
    • diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index be221fd532f..e4e4ad943e2 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1971,56 +1971,54 @@
    - - - - - - - - - - - - Link - - - -
    - -
    -     - - + + + + + + + + + + + + + + + + +
    + +
    +     + diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index b68acaf7488..d94f1e08f32 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -1206,3 +1206,11 @@ span.label-default { background-color: #757575 } min-width: 130px; padding: 0px; } + +#datasetForm .csl-entry { + border-style: outset; + border-radius: 5px; + border-color: #757575; + margin: 10px 0px 10px 0px; + padding: 10px; + } \ No newline at end of file From 1211c2bf6169d37e86d4ac39ca08840afeaab410 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 15 Nov 2024 16:05:43 -0500 Subject: [PATCH 0184/1048] reduce logging level --- .../pidproviders/doi/datacite/DataCiteDOIProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java index 814699c027a..0b1e3ff5e7d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java @@ -385,7 +385,7 @@ public JsonObject getCSLJson(GlobalId doi) { while((current = in.readLine()) != null) { cslString += current; } - logger.info(cslString); + logger.fine(cslString); JsonObject csl = JsonUtil.getJsonObject(cslString); return csl; } catch (IOException | URISyntaxException e) { From 5db131479dbb027de4e95df23a5474cc6fc2ef2a Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 15 Nov 2024 16:06:14 -0500 Subject: [PATCH 0185/1048] style groups, styling, locale --- .../iq/dataverse/settings/JvmSettings.java | 4 ++ .../harvard/iq/dataverse/util/CSLUtil.java | 38 ++++++++++++++++--- src/main/java/propertyFiles/Bundle.properties | 4 +- src/main/webapp/dataset.xhtml | 9 +++-- src/main/webapp/resources/css/structure.css | 11 +++++- 5 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index d7eea970b8a..ff71412a8ad 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -256,6 +256,10 @@ public enum JvmSettings { // STORAGE USE SETTINGS SCOPE_STORAGEUSE(PREFIX, "storageuse"), STORAGEUSE_DISABLE_UPDATES(SCOPE_STORAGEUSE, "disable-storageuse-increments"), + + //CSL CITATION SETTINGS + SCOPE_CSL(PREFIX, "csl"), + COMMON_STYLES(SCOPE_CSL, "common-styles"), ; private static final String SCOPE_SEPARATOR = "."; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/CSLUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/CSLUtil.java index c01be52dc0e..07485a1c302 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/CSLUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/CSLUtil.java @@ -4,23 +4,29 @@ import java.io.IOException; import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.logging.Logger; import de.undercouch.citeproc.CSL; import de.undercouch.citeproc.helper.CSLUtils; +import edu.harvard.iq.dataverse.settings.JvmSettings; import jakarta.ejb.Singleton; +import jakarta.faces.model.SelectItem; +import jakarta.faces.model.SelectItemGroup; public class CSLUtil { private static final Logger logger = Logger.getLogger(CSLUtil.class.getName()); static ArrayList supportedStyles; + static List groupedStyles; - public static List getSupportedStyles() { - if (supportedStyles != null) { - return supportedStyles; + public static List getSupportedStyles(Locale locale) { + if (groupedStyles != null) { + return groupedStyles; } supportedStyles = new ArrayList<>(); try { @@ -33,7 +39,29 @@ public static List getSupportedStyles() { e.printStackTrace(); } supportedStyles.sort(Comparator.naturalOrder()); - return supportedStyles; + + groupedStyles = new ArrayList<>(); + + SelectItemGroup commonStyles = new SelectItemGroup(BundleUtil.getStringFromBundle("dataset.cite.cslDialog.commonStyles", locale)); + String styles = JvmSettings.COMMON_STYLES.lookupOptional().orElse("chicago-author-date, ieee"); + ArrayList commonArray = new ArrayList<>(); + String[] styleStrings = styles.split("%s,%s"); + Arrays.stream(styleStrings).forEach(style -> { + commonArray.add(new SelectItem(style, style)); + supportedStyles.remove(style); + }); + + SelectItemGroup otherStyles = new SelectItemGroup(BundleUtil.getStringFromBundle("dataset.cite.cslDialog.otherStyles", locale)); + + ArrayList otherArray = new ArrayList<>(supportedStyles.size()); + supportedStyles.forEach(style -> { + otherArray.add(new SelectItem(style, style)); + }); + otherStyles.setSelectItems(otherArray); + + groupedStyles.add(commonStyles); + groupedStyles.add(otherStyles); + return groupedStyles; } /** @@ -64,7 +92,7 @@ public static String getCitationFormat(String styleName) throws IOException { // load style String result = CSLUtils.readURLToString(url, "UTF-8"); - result=result.replace("\"", "\\\"").replace("\r","").replace("\n",""); + result = result.replace("\"", "\\\"").replace("\r", "").replace("\n", ""); return result; } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 5d4ec630950..4484556cdad 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1609,7 +1609,9 @@ dataset.cite.downloadBtn.bib=BibTeX dataset.cite.viewCitation=View Styled Citation dataset.cite.cslDialog.title=Styled Citation dataset.cite.cslDialog.select=Select a CSL style -dataset.cite.cslDialog.citation=Citation in {0} style +dataset.cite.cslDialog.commonStyles=Common Styles +dataset.cite.cslDialog.otherStyles=More Styles +dataset.cite.cslDialog.citation=Citation in "{0}" style dataset.cite.cslDialog.generating=Generating Citation... dataset.cite.cslDialog.close=Close dataset.create.authenticatedUsersOnly=Only authenticated users can create datasets. diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index e4e4ad943e2..9256836aa27 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1983,13 +1983,13 @@ styleClass="form-control" style="width:auto !important; max-width:100%; min-width:200px;" filter="true" filterMatchMode="contains"> - - + +