From e34177cafbd5c6fec77cc1a1c2cd301a82f39366 Mon Sep 17 00:00:00 2001 From: Philipp Conzett Date: Sun, 25 May 2025 10:01:11 +0200 Subject: [PATCH 001/634] Add files via upload --- scripts/api/data/licenses/licenseEUPL-1.2.json | 11 +++++++++++ scripts/api/data/licenses/licenseODC-By-1.0.json | 11 +++++++++++ scripts/api/data/licenses/licenseODbL-1.0.json | 11 +++++++++++ scripts/api/data/licenses/licensePDDL-1.0.json | 11 +++++++++++ 4 files changed, 44 insertions(+) create mode 100644 scripts/api/data/licenses/licenseEUPL-1.2.json create mode 100644 scripts/api/data/licenses/licenseODC-By-1.0.json create mode 100644 scripts/api/data/licenses/licenseODbL-1.0.json create mode 100644 scripts/api/data/licenses/licensePDDL-1.0.json diff --git a/scripts/api/data/licenses/licenseEUPL-1.2.json b/scripts/api/data/licenses/licenseEUPL-1.2.json new file mode 100644 index 00000000000..54b62552d1c --- /dev/null +++ b/scripts/api/data/licenses/licenseEUPL-1.2.json @@ -0,0 +1,11 @@ +{ + "name": "EUPL-1.2", + "uri": "https://joinup.ec.europa.eu/page/eupl-text-11-12", + "shortDescription": "European Union Public License 1.2", + "active": true, + "sortOrder": 13, + "rightsIdentifier": "EUPL-1.2", + "rightsIdentifierScheme": "SPDX", + "schemeUri": "https://spdx.org/licenses/", + "languageCode": "en" +} \ No newline at end of file diff --git a/scripts/api/data/licenses/licenseODC-By-1.0.json b/scripts/api/data/licenses/licenseODC-By-1.0.json new file mode 100644 index 00000000000..fdeab439d86 --- /dev/null +++ b/scripts/api/data/licenses/licenseODC-By-1.0.json @@ -0,0 +1,11 @@ +{ + "name": "ODC-By-1.0", + "uri": "https://opendatacommons.org/licenses/by/1.0/", + "shortDescription": "Open Data Commons Attribution License v1.0", + "active": true, + "sortOrder": 12, + "rightsIdentifier": "ODC-By-1.0", + "rightsIdentifierScheme": "SPDX", + "schemeUri": "https://spdx.org/licenses/", + "languageCode": "en" +} \ No newline at end of file diff --git a/scripts/api/data/licenses/licenseODbL-1.0.json b/scripts/api/data/licenses/licenseODbL-1.0.json new file mode 100644 index 00000000000..bf13ad404b4 --- /dev/null +++ b/scripts/api/data/licenses/licenseODbL-1.0.json @@ -0,0 +1,11 @@ +{ + "name": "ODbL-1.0", + "uri": "http://www.opendatacommons.org/licenses/odbl/1.0/", + "shortDescription": "Open Data Commons Open Database License v1.0", + "active": true, + "sortOrder": 11, + "rightsIdentifier": "ODbL-1.0", + "rightsIdentifierScheme": "SPDX", + "schemeUri": "https://spdx.org/licenses/", + "languageCode": "en" +} \ No newline at end of file diff --git a/scripts/api/data/licenses/licensePDDL-1.0.json b/scripts/api/data/licenses/licensePDDL-1.0.json new file mode 100644 index 00000000000..ae01f3d2109 --- /dev/null +++ b/scripts/api/data/licenses/licensePDDL-1.0.json @@ -0,0 +1,11 @@ +{ + "name": "PDDL-1.0", + "uri": "http://opendatacommons.org/licenses/pddl/1.0/", + "shortDescription": "Open Data Commons Public Domain Dedication & License 1.0", + "active": true, + "sortOrder": 12, + "rightsIdentifier": "PDDL-1.0", + "rightsIdentifierScheme": "SPDX", + "schemeUri": "https://spdx.org/licenses/", + "languageCode": "en" +} \ No newline at end of file From f74b832ed160481a90f63344a73f607becae9e45 Mon Sep 17 00:00:00 2001 From: Philipp Conzett Date: Sun, 25 May 2025 11:07:51 +0200 Subject: [PATCH 002/634] Update config.rst --- doc/sphinx-guides/source/installation/config.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index d5fa101b095..d06ee9b242e 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2038,13 +2038,23 @@ JSON files for `Creative Commons licenses `_ are provided below. + +- :download:`licenseODbL-1.0.json <../../../../scripts/api/data/licenses/licenseODbL-1.0.json>` +- :download:`licenseODC-By-1.0.json <../../../../scripts/api/data/licenses/licenseODC-By-1.0.json>` +- :download:`licensePDDL-1.0.json <../../../../scripts/api/data/licenses/licensePDDL-1.0.json>` + Adding Software Licenses ^^^^^^^^^^^^^^^^^^^^^^^^ JSON files for software licenses are provided below. -- :download:`licenseMIT.json <../../../../scripts/api/data/licenses/licenseMIT.json>` - :download:`licenseApache-2.0.json <../../../../scripts/api/data/licenses/licenseApache-2.0.json>` +- :download:`licenseMIT.json <../../../../scripts/api/data/licenses/licenseMIT.json>` +- :download:`licenseEUPL-1.2.json <../../../../scripts/api/data/licenses/licenseEUPL-1.2.json>` Adding Country-Specific Licenses ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From f2747649efa5ce2406e2f985eddfdda3f500375b Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 28 May 2025 10:09:54 -0400 Subject: [PATCH 003/634] make sort order unique #11522 --- scripts/api/data/licenses/licenseEUPL-1.2.json | 4 ++-- scripts/api/data/licenses/licensePDDL-1.0.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/api/data/licenses/licenseEUPL-1.2.json b/scripts/api/data/licenses/licenseEUPL-1.2.json index 54b62552d1c..cb3afad2fea 100644 --- a/scripts/api/data/licenses/licenseEUPL-1.2.json +++ b/scripts/api/data/licenses/licenseEUPL-1.2.json @@ -3,9 +3,9 @@ "uri": "https://joinup.ec.europa.eu/page/eupl-text-11-12", "shortDescription": "European Union Public License 1.2", "active": true, - "sortOrder": 13, + "sortOrder": 14, "rightsIdentifier": "EUPL-1.2", "rightsIdentifierScheme": "SPDX", "schemeUri": "https://spdx.org/licenses/", "languageCode": "en" -} \ No newline at end of file +} diff --git a/scripts/api/data/licenses/licensePDDL-1.0.json b/scripts/api/data/licenses/licensePDDL-1.0.json index ae01f3d2109..986b97e5b01 100644 --- a/scripts/api/data/licenses/licensePDDL-1.0.json +++ b/scripts/api/data/licenses/licensePDDL-1.0.json @@ -3,9 +3,9 @@ "uri": "http://opendatacommons.org/licenses/pddl/1.0/", "shortDescription": "Open Data Commons Public Domain Dedication & License 1.0", "active": true, - "sortOrder": 12, + "sortOrder": 13, "rightsIdentifier": "PDDL-1.0", "rightsIdentifierScheme": "SPDX", "schemeUri": "https://spdx.org/licenses/", "languageCode": "en" -} \ No newline at end of file +} From b6dbf922fb4bc50742240fc9ab9443f529110443 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 28 May 2025 10:16:54 -0400 Subject: [PATCH 004/634] add release note snippet --- doc/release-notes/11522-licenses-added.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/release-notes/11522-licenses-added.md diff --git a/doc/release-notes/11522-licenses-added.md b/doc/release-notes/11522-licenses-added.md new file mode 100644 index 00000000000..0639b86a931 --- /dev/null +++ b/doc/release-notes/11522-licenses-added.md @@ -0,0 +1,13 @@ +### Additional licenses + +The following Open Data Commons licenses have been added: + +- Open Database License (ODbL) +- Open Data Commons Attribution License (ODC-By) +- Open Data Commons Public Domain Dedication and License (PDDL)) + +The following software license has been added: + +- European Union Public License (EUPL) + +See [the guides](https://guides.dataverse.org/en/6.7/installation/config.html#configuring-licenses) and #11522. From 7f6aaad27c02af31e13bfec7d515b8a869ccd0da Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 28 May 2025 10:36:18 -0400 Subject: [PATCH 005/634] add OGL-UK-3.0 license #9403 #11521 --- doc/release-notes/11522-licenses-added.md | 4 ++++ doc/sphinx-guides/source/installation/config.rst | 1 + scripts/api/data/licenses/licenseOGL-UK-3.0.json | 11 +++++++++++ 3 files changed, 16 insertions(+) create mode 100644 scripts/api/data/licenses/licenseOGL-UK-3.0.json diff --git a/doc/release-notes/11522-licenses-added.md b/doc/release-notes/11522-licenses-added.md index 0639b86a931..1a02916f769 100644 --- a/doc/release-notes/11522-licenses-added.md +++ b/doc/release-notes/11522-licenses-added.md @@ -10,4 +10,8 @@ The following software license has been added: - European Union Public License (EUPL) +The following country-specific license has been added: + +- Open Government Licence (OGL UK) + See [the guides](https://guides.dataverse.org/en/6.7/installation/config.html#configuring-licenses) and #11522. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index d06ee9b242e..0a09524abe3 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2060,6 +2060,7 @@ Adding Country-Specific Licenses ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - :download:`licenseEtalab-2.0.json <../../../../scripts/api/data/licenses/licenseEtalab-2.0.json>` used in France (Etalab Open License 2.0, CC-BY 2.0 compliant). +- :download:`licenseOGL-UK-3.0.json <../../../../scripts/api/data/licenses/licenseOGL-UK-3.0.json>` Contributing to the Collection of Standard Licenses Above ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/scripts/api/data/licenses/licenseOGL-UK-3.0.json b/scripts/api/data/licenses/licenseOGL-UK-3.0.json new file mode 100644 index 00000000000..7363645e075 --- /dev/null +++ b/scripts/api/data/licenses/licenseOGL-UK-3.0.json @@ -0,0 +1,11 @@ +{ + "name": "OGL UK 3.0", + "uri": "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3", + "shortDescription": "Open Government Licence v3.0.", + "active": true, + "sortOrder": 15, + "rightsIdentifier": "OGL-UK-3.0", + "rightsIdentifierScheme": "SPDX", + "schemeUri": "https://spdx.org/licenses/", + "languageCode": "en" +} From 7ac49381c58d134a23fb128065def0a05284071a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 28 May 2025 10:38:43 -0400 Subject: [PATCH 006/634] use suggested text in release note snippet #11521 --- doc/release-notes/11522-licenses-added.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/11522-licenses-added.md b/doc/release-notes/11522-licenses-added.md index 1a02916f769..6afbb0fcb2c 100644 --- a/doc/release-notes/11522-licenses-added.md +++ b/doc/release-notes/11522-licenses-added.md @@ -14,4 +14,4 @@ The following country-specific license has been added: - Open Government Licence (OGL UK) -See [the guides](https://guides.dataverse.org/en/6.7/installation/config.html#configuring-licenses) and #11522. +The licenses above are widely recognized and used in Europe and beyond to promote data and software sharing. See [the guides](https://guides.dataverse.org/en/6.7/installation/config.html#configuring-licenses) and #11522. From 9196846820df730722810a352f8d79217600e24f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 28 May 2025 10:51:18 -0400 Subject: [PATCH 007/634] clarify how to add licenses #11521 #10426 --- doc/sphinx-guides/source/installation/config.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 0a09524abe3..97f83fa4a43 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2072,10 +2072,14 @@ If you do not find the license JSON you need above, you are encouraged to contri - Copy an existing license as a starting point. - Name your file using the SPDX identifier. For example, if the identifier is ``Apache-2.0``, you should name your file ``licenseApache-2.0.json``. - For the ``name`` field, use the "short identifier" from the SPDX landing page (e.g. ``Apache-2.0``). -- For the ``description`` field, use the "full name" from the SPDX landing page (e.g. ``Apache License 2.0``). +- For the ``shortDescription`` field, use the "full name" from the SPDX landing page (e.g. ``Apache License 2.0``) followed by a period (full-stop) (e.g. ``Apache License 2.0.``). - For the ``uri`` field, we encourage you to use the same resource that DataCite uses, which is often the same as the first "Other web pages for this license" on the SPDX page for the license. When these differ, or there are other concerns about the URI DataCite uses, please reach out to the community to see if a consensus can be reached. - For the ``active`` field, put ``true``. - For the ``sortOrder`` field, put the next sequential number after checking previous files with ``grep sortOrder scripts/api/data/licenses/*``. +- For the ``rightsIdentifier`` field, use the identifier from SPDX (e.g. ``Apache-2.0``). +- For the ``rightsIdentifierScheme`` field, use "SPDX". +- For the ``schemeUri`` field, use "https://spdx.org/licenses/". +- For the ``languageCode`` field, use "en". Note that prior to Dataverse 6.2, various license above have been added that do not adhere perfectly with this procedure. For example, the ``name`` for the CC0 license is ``CC0 1.0`` (no dash) rather than ``CC0-1.0`` (with a dash). We are keeping the existing names for backward compatibility. For more on standarizing license configuration, see https://github.com/IQSS/dataverse/issues/8512 From 253a0342528ba7103d1aff1dfb23ce756e91bb20 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 8 Apr 2025 13:32:53 -0400 Subject: [PATCH 008/634] basic roleassignment auditing for assign, revoke --- .../dataverse/DataverseRoleServiceBean.java | 50 +++++- .../iq/dataverse/RoleAssignmentAudit.java | 157 ++++++++++++++++++ .../command/impl/AssignRoleCommand.java | 37 +++-- .../CuratePublishedDatasetVersionCommand.java | 2 +- .../impl/DeleteDatasetVersionCommand.java | 2 +- .../command/impl/DeleteRoleCommand.java | 2 +- .../command/impl/RevokeAllRolesCommand.java | 2 +- .../command/impl/RevokeRoleCommand.java | 20 +-- .../iq/dataverse/settings/FeatureFlags.java | 5 + .../impl/DeletePrivateUrlCommandTest.java | 2 +- 10 files changed, 242 insertions(+), 37 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java index b751841da74..00368ff0b67 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java @@ -4,6 +4,7 @@ import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.RoleAssignee; import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.authorization.RoleAssignmentSet; import edu.harvard.iq.dataverse.search.IndexAsync; import edu.harvard.iq.dataverse.search.IndexResponse; @@ -13,7 +14,6 @@ import java.util.Collection; import java.util.HashSet; import java.util.List; -import java.util.Objects; import java.util.Set; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -23,6 +23,7 @@ import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; import jakarta.persistence.TypedQuery; +import edu.harvard.iq.dataverse.settings.FeatureFlags; /** * @@ -87,6 +88,23 @@ public RoleAssignment save(RoleAssignment assignment, boolean createIndex) { } return assignment; } + + + /** + * Saves a RoleAssignmentAudit entry to the database. + * + * @param audit The RoleAssignmentAudit object to be saved + * @return The persisted RoleAssignmentAudit object + */ + public RoleAssignmentAudit saveAudit(RoleAssignmentAudit audit) { + if (audit.getAuditId() == null) { + em.persist(audit); + em.flush(); // Ensure the entity is persisted immediately + } else { + audit = em.merge(audit); + } + return audit; + } private IndexResponse indexDefinitionPoint(DvObject definitionPoint) { /** @@ -135,7 +153,7 @@ public DataverseRole findCustomRoleByAliasAndOwner(String alias, Long ownerId) { .getSingleResult(); } - public void revoke(Set roles, RoleAssignee assignee, DvObject defPoint) { +/* public void revoke(Set roles, RoleAssignee assignee, DvObject defPoint) { for (DataverseRole role : roles) { em.createNamedQuery("RoleAssignment.deleteByAssigneeIdentifier_RoleIdDefinition_PointId") .setParameter("assigneeIdentifier", assignee.getIdentifier()) @@ -146,34 +164,47 @@ public void revoke(Set roles, RoleAssignee assignee, DvObject def } em.refresh(assignee); } - - public void revoke(RoleAssignment ra) { +*/ + public void revoke(RoleAssignment ra, DataverseRequest req) { if (!em.contains(ra)) { ra = em.merge(ra); } + + // Create audit entry if feature flag is set + if (FeatureFlags.ROLE_ASSIGNMENT_AUDITING.enabled()) { + RoleAssignmentAudit audit = new RoleAssignmentAudit(ra, req, RoleAssignmentAudit.ActionType.REVOKE); + saveAudit(audit); + } + em.remove(ra); /** * @todo update permissionModificationTime here. */ indexAsync.indexRole(ra); } - + // "nuclear" remove-all roles for a user or group: // (Note that all the "definition points" - i.e., the dvObjects // on which the roles were assigned - need to be reindexed for permissions // once the role assignments are removed! - public void revokeAll(RoleAssignee assignee) { + public void revokeAll(RoleAssignee assignee, DataverseRequest req) { Set reindexSet = new HashSet<>(); - + for (RoleAssignment ra : roleAssigneeService.getAssignmentsFor(assignee.getIdentifier())) { if (!em.contains(ra)) { ra = em.merge(ra); } + + // Create audit entry if feature flag is set + if (FeatureFlags.ROLE_ASSIGNMENT_AUDITING.enabled()) { + RoleAssignmentAudit audit = new RoleAssignmentAudit(ra, req, RoleAssignmentAudit.ActionType.REVOKE); + saveAudit(audit); + } + em.remove(ra); - reindexSet.add(ra.getDefinitionPoint()); } - + indexAsync.indexRoles(reindexSet); } @@ -329,4 +360,5 @@ For a given permission and dataverse Id get all of the roles (built-in or owned } return retVal; } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java new file mode 100644 index 00000000000..2eafafb6de4 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java @@ -0,0 +1,157 @@ +package edu.harvard.iq.dataverse; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import jakarta.persistence.Temporal; +import jakarta.persistence.TemporalType; +import java.io.Serializable; +import java.util.Date; + +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; + +@Entity +@Table(name = "role_assignment_audit") +public class RoleAssignmentAudit implements Serializable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "audit_id") + private Long auditId; + + @Column(name = "role_assignment_id") + private Long roleAssignmentId; + + @Enumerated(EnumType.STRING) + @Column(name = "action_type", nullable = false) + private ActionType actionType; + + @Temporal(TemporalType.TIMESTAMP) + @Column(name = "action_timestamp", nullable = false) + private Date actionTimestamp; + + @Column(name = "action_by_identifier", nullable = false) + private String actionByIdentifier; + + @Column(name = "assignee_identifier", nullable = false) + private String assigneeIdentifier; + + @Column(name = "role_id") + private Long roleId; + + @Column(name = "role_alias", nullable = false) + private String roleAlias; + + @Column(name = "definition_point_id") + private Long definitionPointId; + + @Column(name = "definition_point_identifier", nullable = false) + private String definitionPointIdentifier; + + public enum ActionType { + ASSIGN, REVOKE + } + + // Constructors + public RoleAssignmentAudit() { + } + + public RoleAssignmentAudit(RoleAssignment roleAssignment, DataverseRequest request, ActionType actionType) { + this.roleAssignmentId = roleAssignment.getId(); + this.actionType = actionType; + this.actionTimestamp = new Date(); + this.actionByIdentifier = request.getUser().getIdentifier(); + this.assigneeIdentifier = roleAssignment.getAssigneeIdentifier(); + this.roleId = roleAssignment.getRole().getId(); + this.roleAlias = roleAssignment.getRole().getAlias(); + this.definitionPointId = roleAssignment.getDefinitionPoint().getId(); + this.definitionPointIdentifier = roleAssignment.getDefinitionPoint().getIdentifier(); + } + + // Getters and setters + public Long getAuditId() { + return auditId; + } + + public void setAuditId(Long auditId) { + this.auditId = auditId; + } + + public Long getRoleAssignmentId() { + return roleAssignmentId; + } + + public void setRoleAssignmentId(Long roleAssignmentId) { + this.roleAssignmentId = roleAssignmentId; + } + + public ActionType getActionType() { + return actionType; + } + + public void setActionType(ActionType actionType) { + this.actionType = actionType; + } + + public Date getActionTimestamp() { + return actionTimestamp; + } + + public void setActionTimestamp(Date actionTimestamp) { + this.actionTimestamp = actionTimestamp; + } + + public String getActionByIdentifier() { + return actionByIdentifier; + } + + public void setActionByIdentifier(String actionByIdentifier) { + this.actionByIdentifier = actionByIdentifier; + } + + public String getAssigneeIdentifier() { + return assigneeIdentifier; + } + + public void setAssigneeIdentifier(String assigneeIdentifier) { + this.assigneeIdentifier = assigneeIdentifier; + } + + public Long getRoleId() { + return roleId; + } + + public void setRoleId(Long roleId) { + this.roleId = roleId; + } + + public String getRoleAlias() { + return roleAlias; + } + + public void setRoleAlias(String roleAlias) { + this.roleAlias = roleAlias; + } + + public Long getDefinitionPointId() { + return definitionPointId; + } + + public void setDefinitionPointId(Long definitionPointId) { + this.definitionPointId = definitionPointId; + } + + public String getDefinitionPointIdentifier() { + return definitionPointIdentifier; + } + + public void setDefinitionPointIdentifier(String definitionPointIdentifier) { + this.definitionPointIdentifier = definitionPointIdentifier; + } + +} \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java index 121af765737..2bd8b6c1090 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java @@ -19,7 +19,10 @@ import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.RoleAssignmentAudit; +import edu.harvard.iq.dataverse.settings.FeatureFlags; import java.util.Collections; +import java.util.Date; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -61,21 +64,29 @@ public AssignRoleCommand(PrivateUrlUser privateUrlUser, DataverseRole memberRole this.anonymizedAccess= anonymizedAccess; } - @Override - public RoleAssignment execute(CommandContext ctxt) throws CommandException { - if (grantee instanceof AuthenticatedUser) { - AuthenticatedUser user = (AuthenticatedUser) grantee; - if (user.isDeactivated()) { - throw new IllegalCommandException("User " + user.getUserIdentifier() + " is deactivated and cannot be given a role.", this); - } - } - if(isExistingRole(ctxt)){ - throw new IllegalCommandException(BundleUtil.getStringFromBundle("datasets.api.grant.role.assignee.has.role.error"), this); +@Override +public RoleAssignment execute(CommandContext ctxt) throws CommandException { + if (grantee instanceof AuthenticatedUser) { + AuthenticatedUser user = (AuthenticatedUser) grantee; + if (user.isDeactivated()) { + throw new IllegalCommandException("User " + user.getUserIdentifier() + " is deactivated and cannot be given a role.", this); } - // TODO make sure the role is defined on the dataverse. - RoleAssignment roleAssignment = new RoleAssignment(role, grantee, defPoint, privateUrlToken, anonymizedAccess); - return ctxt.roles().save(roleAssignment); } + if(isExistingRole(ctxt)){ + throw new IllegalCommandException(BundleUtil.getStringFromBundle("datasets.api.grant.role.assignee.has.role.error"), this); + } + // TODO make sure the role is defined on the dataverse. + RoleAssignment roleAssignment = new RoleAssignment(role, grantee, defPoint, privateUrlToken, anonymizedAccess); + RoleAssignment savedRoleAssignment = ctxt.roles().save(roleAssignment); + + // Check if ROLE_ASSIGNMENT_AUDITING feature flag is enabled + if (FeatureFlags.ROLE_ASSIGNMENT_AUDITING.enabled()) { + RoleAssignmentAudit audit = new RoleAssignmentAudit(savedRoleAssignment, getRequest(), RoleAssignmentAudit.ActionType.ASSIGN); + ctxt.roles().saveAudit(audit); + } + + return savedRoleAssignment; +} private boolean isExistingRole(CommandContext ctxt) { return ctxt.roles() diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index 3629432b7e4..45cee6e9595 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -189,7 +189,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { RoleAssignment ra = ctxt.privateUrl().getPrivateUrlRoleAssignmentFromDataset(savedDataset); if (ra != null) { - ctxt.roles().revoke(ra); + ctxt.roles().revoke(ra, getRequest()); } // And update metadata at PID provider diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDatasetVersionCommand.java index a67d7008ef8..efb49a67b9d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDatasetVersionCommand.java @@ -92,7 +92,7 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { PrivateUrlUser privateUrlUser = new PrivateUrlUser(doomed.getId()); List roleAssignments = ctxt.roles().directRoleAssignments(privateUrlUser, doomed); for (RoleAssignment roleAssignment : roleAssignments) { - ctxt.roles().revoke(roleAssignment); + ctxt.roles().revoke(roleAssignment, getRequest()); } } boolean doNormalSolrDocCleanUp = true; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteRoleCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteRoleCommand.java index b0baf3a24e1..5608eb5af1f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteRoleCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteRoleCommand.java @@ -25,7 +25,7 @@ public DeleteRoleCommand(DataverseRequest aRequest, DataverseRole doomed ) { @Override protected void executeImpl(CommandContext ctxt) throws CommandException { for ( RoleAssignment ra : ctxt.roles().roleAssignments(doomed.getId()) ) { - ctxt.roles().revoke(ra); + ctxt.roles().revoke(ra, getRequest()); } ctxt.roles().delete(doomed.getId()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RevokeAllRolesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RevokeAllRolesCommand.java index e44431591c2..ed16889f736 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RevokeAllRolesCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RevokeAllRolesCommand.java @@ -42,7 +42,7 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { } try { - ctxt.roles().revokeAll(assignee); + ctxt.roles().revokeAll(assignee, getRequest()); ctxt.explicitGroups().revokeAllGroupsForAssignee(assignee); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RevokeRoleCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RevokeRoleCommand.java index 26ab88d29d8..f2899408393 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RevokeRoleCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RevokeRoleCommand.java @@ -19,18 +19,18 @@ */ // no annotations here, since permissions are dynamically decided public class RevokeRoleCommand extends AbstractVoidCommand { - - private final RoleAssignment toBeRevoked; - public RevokeRoleCommand(RoleAssignment toBeRevoked, DataverseRequest aRequest) { + private final RoleAssignment toBeRevoked; + + public RevokeRoleCommand(RoleAssignment toBeRevoked, DataverseRequest aRequest) { super(aRequest, toBeRevoked.getDefinitionPoint()); - this.toBeRevoked = toBeRevoked; - } - - @Override - protected void executeImpl(CommandContext ctxt) throws CommandException { - ctxt.roles().revoke(toBeRevoked); - } + this.toBeRevoked = toBeRevoked; + } + + @Override + protected void executeImpl(CommandContext ctxt) throws CommandException { + ctxt.roles().revoke(toBeRevoked, getRequest()); + } @Override public Map> getRequiredPermissions() { diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java index 27c65ed067c..539e898255c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java @@ -167,6 +167,11 @@ public enum FeatureFlags { */ ADD_LOCAL_CONTEXTS_PERMISSION_CHECK("add-local-contexts-permission-check"), + /** + * This flag turns on auditing of role assignments - keeping a record of when roles were granted + * or revoked, at what times, and by whom. + */ + ROLE_ASSIGNMENT_AUDITING("role-assignment-auditing"), ; final String flag; diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/DeletePrivateUrlCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/DeletePrivateUrlCommandTest.java index 0a4e5ed2d7e..be633a68dba 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/DeletePrivateUrlCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/DeletePrivateUrlCommandTest.java @@ -64,7 +64,7 @@ public List directRoleAssignments(RoleAssignee roas, DvObject dv } @Override - public void revoke(RoleAssignment ra) { + public void revoke(RoleAssignment ra, DataverseRequest req) { // no-op } From a7f4b61c09505ffa92a8b6541b6a9823df1733d0 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 8 Apr 2025 13:35:45 -0400 Subject: [PATCH 009/634] add indexes --- .../edu/harvard/iq/dataverse/RoleAssignmentAudit.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java index 2eafafb6de4..2645f641cc0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java +++ b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java @@ -10,13 +10,22 @@ import jakarta.persistence.Table; import jakarta.persistence.Temporal; import jakarta.persistence.TemporalType; +import jakarta.persistence.Index; import java.io.Serializable; import java.util.Date; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; @Entity -@Table(name = "role_assignment_audit") +@Table(name = "role_assignment_audit", indexes = { + @Index(name = "idx_raa_role_assignment_id", columnList = "role_assignment_id"), + @Index(name = "idx_raa_action_type", columnList = "action_type"), + @Index(name = "idx_raa_action_timestamp", columnList = "action_timestamp"), + @Index(name = "idx_raa_action_by_identifier", columnList = "action_by_identifier"), + @Index(name = "idx_raa_assignee_identifier", columnList = "assignee_identifier"), + @Index(name = "idx_raa_role_id", columnList = "role_id"), + @Index(name = "idx_raa_definition_point_id", columnList = "definition_point_id") +}) public class RoleAssignmentAudit implements Serializable { @Id From bf0beedc2b1bf23329a51e0d14096ebdd447904c Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 8 Apr 2025 13:55:39 -0400 Subject: [PATCH 010/634] missing import --- .../engine/command/impl/DeletePrivateUrlCommandTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/DeletePrivateUrlCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/DeletePrivateUrlCommandTest.java index be633a68dba..640c66b61e4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/DeletePrivateUrlCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/DeletePrivateUrlCommandTest.java @@ -8,6 +8,7 @@ import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser; import edu.harvard.iq.dataverse.engine.TestCommandContext; import edu.harvard.iq.dataverse.engine.TestDataverseEngine; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; From 47e98050eedf41159a783f4a7eedb9f359c63a16 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 8 Apr 2025 14:34:47 -0400 Subject: [PATCH 011/634] refresh assignment to get id --- .../harvard/iq/dataverse/DataverseRoleServiceBean.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java index 00368ff0b67..17938d900a3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java @@ -77,15 +77,16 @@ public RoleAssignment save(RoleAssignment assignment) { public RoleAssignment save(RoleAssignment assignment, boolean createIndex) { if (assignment.getId() == null) { em.persist(assignment); + em.flush(); // Force synchronization with the database + em.refresh(assignment); // Refresh the entity to ensure it has the latest state, including the ID } else { assignment = em.merge(assignment); } - /** - * @todo update permissionModificationTime here. - */ - if ( createIndex ) { + + if (createIndex) { indexAsync.indexRole(assignment); } + return assignment; } From 1d3a1119e8bb3fade8e27c2275e57daa3ac43a3a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 8 Apr 2025 15:33:29 -0400 Subject: [PATCH 012/634] history panel --- .../iq/dataverse/ManagePermissionsPage.java | 100 ++++++++++++++++++ .../iq/dataverse/RoleAssignmentAudit.java | 7 +- src/main/java/propertyFiles/Bundle.properties | 7 ++ src/main/webapp/permissions-manage.xhtml | 34 ++++++ 4 files changed, 147 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index 0e277c5aa32..5359079ad3f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -26,9 +26,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.Date; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -230,6 +233,44 @@ public void cloneRole(String roleId) { public void editRole(String roleId) { setRole(roleService.find(Long.parseLong(roleId))); } + + /** Role Assignment History */ + private List roleAssignmentHistory; + + public List getRoleAssignmentHistory() { + if (roleAssignmentHistory == null) { + roleAssignmentHistory = new ArrayList<>(); + + List audits = em.createNamedQuery("RoleAssignmentAudit.findByDefinitionPointId", RoleAssignmentAudit.class) + .setParameter("definitionPointId", dvObject.getId()) + .getResultList(); + + Map historyMap = new HashMap<>(); + + for (RoleAssignmentAudit audit : audits) { + Long roleAssignmentId = audit.getRoleAssignmentId(); + RoleAssignmentHistoryEntry entry = historyMap.get(roleAssignmentId); + + if (entry == null) { + entry = new RoleAssignmentHistoryEntry(audit.getAssigneeIdentifier(), audit.getRoleAlias()); + historyMap.put(roleAssignmentId, entry); + } + + if (audit.getActionType() == RoleAssignmentAudit.ActionType.ASSIGN) { + entry.setAssignedBy(audit.getActionByIdentifier()); + entry.setAssignedAt(audit.getActionTimestamp()); + } else if (audit.getActionType() == RoleAssignmentAudit.ActionType.REVOKE) { + entry.setRevokedBy(audit.getActionByIdentifier()); + entry.setRevokedAt(audit.getActionTimestamp()); + } + } + + roleAssignmentHistory.addAll(historyMap.values()); + roleAssignmentHistory.sort(Comparator.comparing(RoleAssignmentHistoryEntry::getAssignedAt).reversed()); + } + return roleAssignmentHistory; + } + /* ============================================================================ @@ -716,4 +757,63 @@ public Long getId() { } } + + public static class RoleAssignmentHistoryEntry { + private String roleName; + private String assigneeIdentifier; + private String assignedBy; + private Date assignedAt; + private String revokedBy; + private Date revokedAt; + + public RoleAssignmentHistoryEntry(String assigneeIdentifier, String roleName) { + this.roleName = roleName; + this.assigneeIdentifier = assigneeIdentifier; + ; + } + + public void setRevokedAt(Date actionTimestamp) { + revokedAt = actionTimestamp; + + } + + public void setRevokedBy(String actionByIdentifier) { + revokedBy = actionByIdentifier; + + } + + public void setAssignedAt(Date actionTimestamp) { + assignedAt = actionTimestamp; + + } + + public void setAssignedBy(String actionByIdentifier) { + assignedBy = actionByIdentifier; + + } + + public String getRoleName() { + return roleName; + } + + public String getAssigneeIdentifier() { + return assigneeIdentifier; + } + + public String getAssignedBy() { + return assignedBy; + } + + public Date getAssignedAt() { + return assignedAt; + } + + public String getRevokedBy() { + return revokedBy; + } + + public Date getRevokedAt() { + return revokedAt; + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java index 2645f641cc0..2209977b541 100644 --- a/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java +++ b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java @@ -11,6 +11,8 @@ import jakarta.persistence.Temporal; import jakarta.persistence.TemporalType; import jakarta.persistence.Index; +import jakarta.persistence.NamedQuery; + import java.io.Serializable; import java.util.Date; @@ -26,6 +28,9 @@ @Index(name = "idx_raa_role_id", columnList = "role_id"), @Index(name = "idx_raa_definition_point_id", columnList = "definition_point_id") }) +@NamedQuery(name = "RoleAssignmentAudit.findByDefinitionPointId", +query = "SELECT ra FROM RoleAssignmentAudit ra WHERE ra.definitionPointId = :definitionPointId ORDER BY ra.roleAssignmenId, ra.actionTimestamp DESC") + public class RoleAssignmentAudit implements Serializable { @Id @@ -79,7 +84,7 @@ public RoleAssignmentAudit(RoleAssignment roleAssignment, DataverseRequest reque this.roleId = roleAssignment.getRole().getId(); this.roleAlias = roleAssignment.getRole().getAlias(); this.definitionPointId = roleAssignment.getDefinitionPoint().getId(); - this.definitionPointIdentifier = roleAssignment.getDefinitionPoint().getIdentifier(); + this.definitionPointIdentifier = roleAssignment.getDefinitionPoint().getGlobalId().asString(); } // Getters and setters diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 05421179d7e..95341c74545 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1203,6 +1203,13 @@ dataverse.permissions.roles.edit=Edit Role dataverse.permissions.roles.copy=Copy Role dataverse.permissions.roles.alias.required=Please enter a unique identifier for this role. dataverse.permissions.roles.name.required=Please enter a name for this role. +dataverse.permissions.history=Role Assignment History +dataverse.permissions.history.description=View the history of role assignments and revocations +dataverse.permissions.history.role=Role +dataverse.permissions.history.assignee=Assignee +dataverse.permissions.history.assigned=Assigned +dataverse.permissions.history.revoked=Revoked +dataverse.permissions.history.notRevoked=Not Revoked # permissions-manage-files.xhtml dataverse.permissionsFiles.title=Restricted File Permissions diff --git a/src/main/webapp/permissions-manage.xhtml b/src/main/webapp/permissions-manage.xhtml index b328e4abc81..e3e5e4ecf6c 100644 --- a/src/main/webapp/permissions-manage.xhtml +++ b/src/main/webapp/permissions-manage.xhtml @@ -197,6 +197,40 @@ + +
+
+ #{bundle['dataverse.permissions.history']} + #{bundle['dataverse.permissions.history.description']} +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + +
+
+
From 2fbde58c32b026577f6c5059918bc2c6f3709002 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 8 Apr 2025 15:46:17 -0400 Subject: [PATCH 013/634] typo --- src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java index 2209977b541..8c64583e814 100644 --- a/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java +++ b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java @@ -29,7 +29,7 @@ @Index(name = "idx_raa_definition_point_id", columnList = "definition_point_id") }) @NamedQuery(name = "RoleAssignmentAudit.findByDefinitionPointId", -query = "SELECT ra FROM RoleAssignmentAudit ra WHERE ra.definitionPointId = :definitionPointId ORDER BY ra.roleAssignmenId, ra.actionTimestamp DESC") +query = "SELECT ra FROM RoleAssignmentAudit ra WHERE ra.definitionPointId = :definitionPointId ORDER BY ra.roleAssignmentId, ra.actionTimestamp DESC") public class RoleAssignmentAudit implements Serializable { From 9da8ee9c36be79c4883d5c75998c46c05fef54a5 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 9 Apr 2025 11:58:02 -0400 Subject: [PATCH 014/634] file RAs --- .../dataverse/ManageFilePermissionsPage.java | 37 ++++++++++++++++++ .../iq/dataverse/ManagePermissionsPage.java | 19 +++++---- .../iq/dataverse/RoleAssignmentAudit.java | 12 ++++-- src/main/java/propertyFiles/Bundle.properties | 2 +- .../webapp/permissions-manage-files.xhtml | 39 +++++++++++++++++++ src/main/webapp/permissions-manage.xhtml | 9 +++-- 6 files changed, 103 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java index 1ead0b13cdc..4cfdeaeaac3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java @@ -5,6 +5,7 @@ */ package edu.harvard.iq.dataverse; +import edu.harvard.iq.dataverse.ManagePermissionsPage.RoleAssignmentHistoryEntry; import edu.harvard.iq.dataverse.api.Util; import edu.harvard.iq.dataverse.authorization.AuthenticationProvider; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; @@ -151,6 +152,7 @@ public String init() { return permissionsWrapper.notAuthorized(); } initMaps(); + roleAssignmentHistory = null; return ""; } @@ -537,6 +539,41 @@ private boolean assignRole(RoleAssignee ra, DataFile file, DataverseRole r) { return true; } + private List roleAssignmentHistory; + + public List getRoleAssignmentHistory() { + if (roleAssignmentHistory == null) { + roleAssignmentHistory = new ArrayList<>(); + + List audits = em.createNamedQuery("RoleAssignmentAudit.findByOwnerId", RoleAssignmentAudit.class) + .setParameter("datasetId", dataset.getId()) + .getResultList(); + + Map historyMap = new HashMap<>(); + + for (RoleAssignmentAudit audit : audits) { + Long roleAssignmentId = audit.getRoleAssignmentId(); + RoleAssignmentHistoryEntry entry = historyMap.get(roleAssignmentId); + + if (entry == null) { + entry = new RoleAssignmentHistoryEntry(audit.getAssigneeIdentifier(), audit.getRoleAlias(), audit.getDefinitionPointId() ); + historyMap.put(roleAssignmentId, entry); + } + + if (audit.getActionType() == RoleAssignmentAudit.ActionType.ASSIGN) { + entry.setAssignedBy(audit.getActionByIdentifier()); + entry.setAssignedAt(audit.getActionTimestamp()); + } else if (audit.getActionType() == RoleAssignmentAudit.ActionType.REVOKE) { + entry.setRevokedBy(audit.getActionByIdentifier()); + entry.setRevokedAt(audit.getActionTimestamp()); + } + } + + roleAssignmentHistory.addAll(historyMap.values()); + roleAssignmentHistory.sort(Comparator.comparing(RoleAssignmentHistoryEntry::getAssignedAt).reversed()); + } + return roleAssignmentHistory; + } boolean renderUserGroupMessages = false; boolean renderFileMessages = false; diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index 5359079ad3f..9b2b04c33a6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -252,7 +252,7 @@ public List getRoleAssignmentHistory() { RoleAssignmentHistoryEntry entry = historyMap.get(roleAssignmentId); if (entry == null) { - entry = new RoleAssignmentHistoryEntry(audit.getAssigneeIdentifier(), audit.getRoleAlias()); + entry = new RoleAssignmentHistoryEntry(audit.getAssigneeIdentifier(), audit.getRoleAlias(), audit.getDefinitionPointId()); historyMap.put(roleAssignmentId, entry); } @@ -765,31 +765,28 @@ public static class RoleAssignmentHistoryEntry { private Date assignedAt; private String revokedBy; private Date revokedAt; + private Long definitionPointId; // New field - public RoleAssignmentHistoryEntry(String assigneeIdentifier, String roleName) { + public RoleAssignmentHistoryEntry(String assigneeIdentifier, String roleName, Long definitionPointId) { this.roleName = roleName; this.assigneeIdentifier = assigneeIdentifier; - ; + this.definitionPointId = definitionPointId; } public void setRevokedAt(Date actionTimestamp) { revokedAt = actionTimestamp; - } public void setRevokedBy(String actionByIdentifier) { revokedBy = actionByIdentifier; - } public void setAssignedAt(Date actionTimestamp) { assignedAt = actionTimestamp; - } public void setAssignedBy(String actionByIdentifier) { assignedBy = actionByIdentifier; - } public String getRoleName() { @@ -815,5 +812,13 @@ public String getRevokedBy() { public Date getRevokedAt() { return revokedAt; } + + public Long getDefinitionPointId() { + return definitionPointId; + } + + public void setDefinitionPointId(Long definitionPointId) { + this.definitionPointId = definitionPointId; + } } } diff --git a/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java index 8c64583e814..dc6fb1806c2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java +++ b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java @@ -11,6 +11,7 @@ import jakarta.persistence.Temporal; import jakarta.persistence.TemporalType; import jakarta.persistence.Index; +import jakarta.persistence.NamedQueries; import jakarta.persistence.NamedQuery; import java.io.Serializable; @@ -28,9 +29,14 @@ @Index(name = "idx_raa_role_id", columnList = "role_id"), @Index(name = "idx_raa_definition_point_id", columnList = "definition_point_id") }) -@NamedQuery(name = "RoleAssignmentAudit.findByDefinitionPointId", -query = "SELECT ra FROM RoleAssignmentAudit ra WHERE ra.definitionPointId = :definitionPointId ORDER BY ra.roleAssignmentId, ra.actionTimestamp DESC") - +@NamedQueries({ + @NamedQuery(name = "RoleAssignmentAudit.findByDefinitionPointId", + query = "SELECT ra FROM RoleAssignmentAudit ra WHERE ra.definitionPointId = :definitionPointId ORDER BY ra.roleAssignmentId, ra.actionTimestamp DESC"), + @NamedQuery(name = "RoleAssignmentAudit.findByOwnerId", + query = "SELECT ra FROM RoleAssignmentAudit ra JOIN DvObject d ON ra.definitionPointId = d.id " + + "WHERE d.owner.id = :datasetId " + + "ORDER BY ra.roleAssignmentId, ra.actionTimestamp DESC") +}) public class RoleAssignmentAudit implements Serializable { @Id diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 95341c74545..095d2ff91e4 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1210,7 +1210,7 @@ dataverse.permissions.history.assignee=Assignee dataverse.permissions.history.assigned=Assigned dataverse.permissions.history.revoked=Revoked dataverse.permissions.history.notRevoked=Not Revoked - +dataverse.permissions.history.file=File Id # permissions-manage-files.xhtml dataverse.permissionsFiles.title=Restricted File Permissions dataverse.permissionsFiles.usersOrGroups=Users/Groups diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index 4e4e56f2051..2be43b776b5 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -218,6 +218,45 @@ + + +
+
+ #{bundle['dataverse.permissions.history']} + #{bundle['dataverse.permissions.history.description']} +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
diff --git a/src/main/webapp/permissions-manage.xhtml b/src/main/webapp/permissions-manage.xhtml index e3e5e4ecf6c..fbd37f5ffcd 100644 --- a/src/main/webapp/permissions-manage.xhtml +++ b/src/main/webapp/permissions-manage.xhtml @@ -104,7 +104,7 @@
#{bundle['dataverse.permissions.usersOrGroups.assignBtn']} @@ -198,7 +198,8 @@
-
+ +
#{bundle['dataverse.permissions.history']} #{bundle['dataverse.permissions.history.description']} @@ -240,7 +241,7 @@
From fe1f3e7add4873d437bbab47645ceae13d059832 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 9 Apr 2025 12:43:20 -0400 Subject: [PATCH 015/634] minor fixes --- src/main/webapp/permissions-manage-files.xhtml | 2 +- src/main/webapp/permissions-manage.xhtml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index 2be43b776b5..c50b2cffccb 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -229,7 +229,7 @@
- + diff --git a/src/main/webapp/permissions-manage.xhtml b/src/main/webapp/permissions-manage.xhtml index fbd37f5ffcd..98dfb240bee 100644 --- a/src/main/webapp/permissions-manage.xhtml +++ b/src/main/webapp/permissions-manage.xhtml @@ -5,6 +5,7 @@ xmlns:ui="http://java.sun.com/jsf/facelets" xmlns:p="http://primefaces.org/ui" xmlns:c="http://java.sun.com/jsp/jstl/core" + xmlns:o="http://omnifaces.org/ui" xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs"> From a1d5c8afe57ceeba6087b99206b35e966be4eb4c Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 9 Apr 2025 14:12:45 -0400 Subject: [PATCH 016/634] sync tables except def point --- src/main/java/propertyFiles/Bundle.properties | 9 ++++---- .../webapp/permissions-manage-files.xhtml | 5 +---- src/main/webapp/permissions-manage.xhtml | 22 +++++++++++-------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 095d2ff91e4..1d7e45c5479 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1207,10 +1207,11 @@ dataverse.permissions.history=Role Assignment History dataverse.permissions.history.description=View the history of role assignments and revocations dataverse.permissions.history.role=Role dataverse.permissions.history.assignee=Assignee -dataverse.permissions.history.assigned=Assigned -dataverse.permissions.history.revoked=Revoked -dataverse.permissions.history.notRevoked=Not Revoked -dataverse.permissions.history.file=File Id +dataverse.permissions.history.assignedBy=Assigned By +dataverse.permissions.history.assignedAt=Assignment Date +dataverse.permissions.history.revokedBy=Revoked By +dataverse.permissions.history.revokedAt=Revocation Date +dataverse.permissions.history.files={0} Files # permissions-manage-files.xhtml dataverse.permissionsFiles.title=Restricted File Permissions dataverse.permissionsFiles.usersOrGroups=Users/Groups diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index c50b2cffccb..64783288df1 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -232,10 +232,7 @@ - - - - + diff --git a/src/main/webapp/permissions-manage.xhtml b/src/main/webapp/permissions-manage.xhtml index 98dfb240bee..9c7b678167b 100644 --- a/src/main/webapp/permissions-manage.xhtml +++ b/src/main/webapp/permissions-manage.xhtml @@ -208,26 +208,30 @@
+ + + - - + + - + - + + - - - - + + + + + -
From 52e18779d89c478c2caf14f2048711aa201c3301 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 9 Apr 2025 14:39:38 -0400 Subject: [PATCH 017/634] aggregate multiple def pts, show tool tip with list --- .../iq/dataverse/ManageFilePermissionsPage.java | 14 ++++++++++++++ .../iq/dataverse/ManagePermissionsPage.java | 13 +++++++------ src/main/java/propertyFiles/Bundle.properties | 1 + src/main/webapp/permissions-manage-files.xhtml | 7 +++++++ 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java index 4cfdeaeaac3..c7adc5368b8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java @@ -568,6 +568,20 @@ public List getRoleAssignmentHistory() { entry.setRevokedAt(audit.getActionTimestamp()); } } + // Second pass: Combine entries with matching criteria + Map finalHistoryMap = new HashMap<>(); + for (RoleAssignmentHistoryEntry entry : historyMap.values()) { + String key = entry.getAssigneeIdentifier() + "|" + entry.getRoleName() + "|" + + entry.getAssignedBy() + "|" + entry.getAssignedAt() + "|" + + entry.getRevokedBy() + "|" + entry.getRevokedAt(); + + RoleAssignmentHistoryEntry existingEntry = finalHistoryMap.get(key); + if (existingEntry == null) { + finalHistoryMap.put(key, entry); + } else { + existingEntry.addDefinitionPointId(entry.getDefinitionPointIds().get(0)); + } + } roleAssignmentHistory.addAll(historyMap.values()); roleAssignmentHistory.sort(Comparator.comparing(RoleAssignmentHistoryEntry::getAssignedAt).reversed()); diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index 9b2b04c33a6..89407c68378 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -765,12 +765,13 @@ public static class RoleAssignmentHistoryEntry { private Date assignedAt; private String revokedBy; private Date revokedAt; - private Long definitionPointId; // New field + private List definitionPointIds; // New field public RoleAssignmentHistoryEntry(String assigneeIdentifier, String roleName, Long definitionPointId) { this.roleName = roleName; this.assigneeIdentifier = assigneeIdentifier; - this.definitionPointId = definitionPointId; + this.definitionPointIds = new ArrayList(); + definitionPointIds.add(definitionPointId); } public void setRevokedAt(Date actionTimestamp) { @@ -813,12 +814,12 @@ public Date getRevokedAt() { return revokedAt; } - public Long getDefinitionPointId() { - return definitionPointId; + public List getDefinitionPointIds() { + return definitionPointIds; } - public void setDefinitionPointId(Long definitionPointId) { - this.definitionPointId = definitionPointId; + public void addDefinitionPointId(Long definitionPointId) { + definitionPointIds.add(definitionPointId); } } } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 1d7e45c5479..10cf777daf5 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1211,6 +1211,7 @@ dataverse.permissions.history.assignedBy=Assigned By dataverse.permissions.history.assignedAt=Assignment Date dataverse.permissions.history.revokedBy=Revoked By dataverse.permissions.history.revokedAt=Revocation Date +dataverse.permissions.history.definedOn=Defined On dataverse.permissions.history.files={0} Files # permissions-manage-files.xhtml dataverse.permissionsFiles.title=Restricted File Permissions diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index 64783288df1..4ead7ddec5b 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -234,6 +234,13 @@ + + + + + + From 8b93f309565b29985be9ccf98430f9ba15734cb9 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 9 Apr 2025 15:05:48 -0400 Subject: [PATCH 018/634] fixes --- .../edu/harvard/iq/dataverse/ManagePermissionsPage.java | 8 ++++++++ src/main/webapp/permissions-manage-files.xhtml | 2 +- src/main/webapp/permissions-manage.xhtml | 3 --- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index 89407c68378..d4bc281a89a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -35,6 +35,8 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; + import jakarta.ejb.EJB; import jakarta.faces.application.FacesMessage; import jakarta.faces.event.ActionEvent; @@ -821,5 +823,11 @@ public List getDefinitionPointIds() { public void addDefinitionPointId(Long definitionPointId) { definitionPointIds.add(definitionPointId); } + + public String getDefinitionPointIdsAsString(List definitionPointIds) { + return definitionPointIds.stream() + .map(Object::toString) + .collect(Collectors.joining(", ")); + } } } diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index 4ead7ddec5b..8ef11965b35 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -236,7 +236,7 @@ + title="#{historyEntry.definitionPointIdsAsString}"> diff --git a/src/main/webapp/permissions-manage.xhtml b/src/main/webapp/permissions-manage.xhtml index 9c7b678167b..dfab2a067a3 100644 --- a/src/main/webapp/permissions-manage.xhtml +++ b/src/main/webapp/permissions-manage.xhtml @@ -214,9 +214,6 @@ - - - From d16eb77577f0ac99b033c7999c02144a664670a3 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 10 Apr 2025 09:17:39 -0400 Subject: [PATCH 019/634] remove param --- .../java/edu/harvard/iq/dataverse/ManagePermissionsPage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index d4bc281a89a..8f8400fe8ca 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -824,7 +824,7 @@ public void addDefinitionPointId(Long definitionPointId) { definitionPointIds.add(definitionPointId); } - public String getDefinitionPointIdsAsString(List definitionPointIds) { + public String getDefinitionPointIdsAsString() { return definitionPointIds.stream() .map(Object::toString) .collect(Collectors.joining(", ")); From c622259b2626df70a111f1bfd619905a66335bad Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 10 Apr 2025 10:51:54 -0400 Subject: [PATCH 020/634] use final map to avoid duplicates --- .../edu/harvard/iq/dataverse/ManageFilePermissionsPage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java index c7adc5368b8..0dbcba1fc97 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java @@ -583,7 +583,7 @@ public List getRoleAssignmentHistory() { } } - roleAssignmentHistory.addAll(historyMap.values()); + roleAssignmentHistory.addAll(finalHistoryMap.values()); roleAssignmentHistory.sort(Comparator.comparing(RoleAssignmentHistoryEntry::getAssignedAt).reversed()); } return roleAssignmentHistory; From 334d8e01a46258265b6fc3679b922d3054ec6b1c Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 10 Apr 2025 10:52:10 -0400 Subject: [PATCH 021/634] use h:outputFormat --- src/main/webapp/permissions-manage-files.xhtml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index 8ef11965b35..9e43d4ce910 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -235,11 +235,11 @@ - - + From 04ac9bad4fa8b1e82802bcbc2a5900da9d7d3b6b Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 14 Apr 2025 14:04:19 -0400 Subject: [PATCH 022/634] Drop refresh --- .../java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java index 17938d900a3..8d31ff907b8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java @@ -78,7 +78,6 @@ public RoleAssignment save(RoleAssignment assignment, boolean createIndex) { if (assignment.getId() == null) { em.persist(assignment); em.flush(); // Force synchronization with the database - em.refresh(assignment); // Refresh the entity to ensure it has the latest state, including the ID } else { assignment = em.merge(assignment); } From c7251f58096cd98120591b18b35cfbae19ba9296 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 14 Apr 2025 14:23:56 -0400 Subject: [PATCH 023/634] move RA to postFlush in create --- .../command/impl/CreateNewDatasetCommand.java | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java index c22a2cdb4a2..5aa2ceb49e4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java @@ -107,9 +107,23 @@ protected void handlePid(Dataset theDataset, CommandContext ctxt) throws Command @Override protected void postPersist( Dataset theDataset, CommandContext ctxt ){ + + + if ( template != null ) { + ctxt.templates().incrementUsageCount(template.getId()); + } + } + + /* Saves role assignments for the dataset. + * Emails those able to publish the dataset (except the creator themselves who already gets an email) + * that a new dataset exists. + * NB: These need the dataset id so have to be postDBFlush (vs postPersist()) + */ + protected void postDBFlush( Dataset theDataset, CommandContext ctxt ){ // set the role to be default contributor role for its dataverse String privateUrlToken = null; if (theDataset.getOwner().getDefaultContributorRole() != null) { + logger.info("New Dataset id: " + theDataset.getId()); RoleAssignment roleAssignment = new RoleAssignment(theDataset.getOwner().getDefaultContributorRole(), getRequest().getUser(), theDataset, privateUrlToken); ctxt.roles().save(roleAssignment, false); @@ -126,17 +140,6 @@ protected void postPersist( Dataset theDataset, CommandContext ctxt ){ // linked here (?) theDataset.setPermissionModificationTime(getTimestamp()); } - - if ( template != null ) { - ctxt.templates().incrementUsageCount(template.getId()); - } - } - - /* Emails those able to publish the dataset (except the creator themselves who already gets an email) - * that a new dataset exists. - * NB: Needs dataset id so has to be postDBFlush (vs postPersist()) - */ - protected void postDBFlush( Dataset theDataset, CommandContext ctxt ){ if(ctxt.settings().isTrueForKey(SettingsServiceBean.Key.SendNotificationOnDatasetCreation, false)) { //QDR - alert curators that a dataset has been created //Should this create a notification too? (which would let us use the notification mailcapbilities to generate the subject/body. From 614697b24193cc6e33033db4aaa0b7296aff97fb Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 14 Apr 2025 16:56:37 -0400 Subject: [PATCH 024/634] move audit code to service --- .../iq/dataverse/DataverseRoleServiceBean.java | 14 ++++++++++---- .../harvard/iq/dataverse/DataverseServiceBean.java | 6 +++--- .../java/edu/harvard/iq/dataverse/api/Admin.java | 5 +++-- .../engine/command/impl/AssignRoleCommand.java | 8 +------- .../command/impl/CreateDataverseCommand.java | 6 +++--- .../command/impl/CreateNewDatasetCommand.java | 2 +- .../command/impl/CreateDataverseCommandTest.java | 6 +++--- .../command/impl/CreatePrivateUrlCommandTest.java | 3 ++- .../command/impl/MoveDatasetCommandTest.java | 6 +++--- .../impl/ReturnDatasetToAuthorCommandTest.java | 2 +- .../impl/SubmitDatasetForReviewCommandTest.java | 2 +- 11 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java index 8d31ff907b8..09097be9c8e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java @@ -70,11 +70,11 @@ public DataverseRole save(DataverseRole aRole) { return aRole; } - public RoleAssignment save(RoleAssignment assignment) { - return save(assignment, true); + public RoleAssignment save(RoleAssignment assignment, DataverseRequest req) { + return save(assignment, true, req); } - public RoleAssignment save(RoleAssignment assignment, boolean createIndex) { + public RoleAssignment save(RoleAssignment assignment, boolean createIndex, DataverseRequest req) { if (assignment.getId() == null) { em.persist(assignment); em.flush(); // Force synchronization with the database @@ -86,6 +86,12 @@ public RoleAssignment save(RoleAssignment assignment, boolean createIndex) { indexAsync.indexRole(assignment); } + // Check if ROLE_ASSIGNMENT_AUDITING feature flag is enabled + if (FeatureFlags.ROLE_ASSIGNMENT_AUDITING.enabled()) { + RoleAssignmentAudit audit = new RoleAssignmentAudit(assignment, req, RoleAssignmentAudit.ActionType.ASSIGN); + saveAudit(audit); + } + return assignment; } @@ -96,7 +102,7 @@ public RoleAssignment save(RoleAssignment assignment, boolean createIndex) { * @param audit The RoleAssignmentAudit object to be saved * @return The persisted RoleAssignmentAudit object */ - public RoleAssignmentAudit saveAudit(RoleAssignmentAudit audit) { + private RoleAssignmentAudit saveAudit(RoleAssignmentAudit audit) { if (audit.getAuditId() == null) { em.persist(audit); em.flush(); // Ensure the entity is persisted immediately diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index f89e707cc03..06ea6969200 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -771,7 +771,7 @@ public List findAllDataverseDatasetChildren(Long dvId) { } public String addRoleAssignmentsToChildren(Dataverse owner, ArrayList rolesToInherit, - boolean inheritAllRoles) { + boolean inheritAllRoles, DataverseRequest req) { /* * This query recursively finds all Dataverses that are inside/children of the * specified one. It recursively finds dvobjects of dtype 'Dataverse' whose @@ -857,7 +857,7 @@ public String addRoleAssignmentsToChildren(Dataverse owner, ArrayList ro try { RoleAssignment ra = new RoleAssignment(inheritableRole, roleUser, childDv, privateUrlToken); if (!existingRAs.get(childDv.getId()).contains(ra)) { - rolesService.save(ra); + rolesService.save(ra, req); } } catch (Exception e) { logger.warning("Unable to assign " + roleAssignment.getAssigneeIdentifier() @@ -877,7 +877,7 @@ public String addRoleAssignmentsToChildren(Dataverse owner, ArrayList ro RoleAssignment ra = new RoleAssignment(inheritableRole, roleGroup, childDv, privateUrlToken); if (!existingRAs.get(childDv.getId()).contains(ra)) { - rolesService.save(ra); + rolesService.save(ra, req); } } catch (Exception e) { logger.warning("Unable to assign " + roleAssignment.getAssigneeIdentifier() diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index ac52b5d9fbf..4cbd405d2cf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -2156,8 +2156,9 @@ public Response addRoleAssignementsToChildren(@Context ContainerRequestContext c if (owner == null) { return error(Response.Status.NOT_FOUND, "Could not find dataverse based on alias supplied: " + alias + "."); } + AuthenticatedUser user = null; try { - AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc); + user = getRequestAuthenticatedUserOrDie(crc); if (!user.isSuperuser()) { return error(Response.Status.FORBIDDEN, "Superusers only."); } @@ -2172,7 +2173,7 @@ public Response addRoleAssignementsToChildren(@Context ContainerRequestContext c if (rolesToInherit.contains("*")) { inheritAllRoles = true; } - return ok(dataverseSvc.addRoleAssignmentsToChildren(owner, rolesToInherit, inheritAllRoles)); + return ok(dataverseSvc.addRoleAssignmentsToChildren(owner, rolesToInherit, inheritAllRoles, createDataverseRequest(user))); } } return error(Response.Status.BAD_REQUEST, diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java index 2bd8b6c1090..7260b433ddb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java @@ -77,13 +77,7 @@ public RoleAssignment execute(CommandContext ctxt) throws CommandException { } // TODO make sure the role is defined on the dataverse. RoleAssignment roleAssignment = new RoleAssignment(role, grantee, defPoint, privateUrlToken, anonymizedAccess); - RoleAssignment savedRoleAssignment = ctxt.roles().save(roleAssignment); - - // Check if ROLE_ASSIGNMENT_AUDITING feature flag is enabled - if (FeatureFlags.ROLE_ASSIGNMENT_AUDITING.enabled()) { - RoleAssignmentAudit audit = new RoleAssignmentAudit(savedRoleAssignment, getRequest(), RoleAssignmentAudit.ActionType.ASSIGN); - ctxt.roles().saveAudit(audit); - } + RoleAssignment savedRoleAssignment = ctxt.roles().save(roleAssignment, getRequest()); return savedRoleAssignment; } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java index 145cfb6199c..d85e82844f5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java @@ -95,7 +95,7 @@ protected Dataverse innerExecute(CommandContext ctxt) throws IllegalCommandExcep DataverseRole adminRole = ctxt.roles().findBuiltinRoleByAlias(DataverseRole.ADMIN); String privateUrlToken = null; - ctxt.roles().save(new RoleAssignment(adminRole, getRequest().getUser(), managedDv, privateUrlToken), false); + ctxt.roles().save(new RoleAssignment(adminRole, getRequest().getUser(), managedDv, privateUrlToken), false, getRequest()); // Add additional role assignments if inheritance is set boolean inheritAllRoles = false; String rolesString = ctxt.settings().getValueForKey(SettingsServiceBean.Key.InheritParentRoleAssignments, ""); @@ -120,13 +120,13 @@ protected Dataverse innerExecute(CommandContext ctxt) throws IllegalCommandExcep if (identifier.startsWith(AuthenticatedUser.IDENTIFIER_PREFIX)) { identifier = identifier.substring(AuthenticatedUser.IDENTIFIER_PREFIX.length()); ctxt.roles().save(new RoleAssignment(role.getRole(), - ctxt.authentication().getAuthenticatedUser(identifier), managedDv, privateUrlToken), false); + ctxt.authentication().getAuthenticatedUser(identifier), managedDv, privateUrlToken), false, getRequest()); } else if (identifier.startsWith(Group.IDENTIFIER_PREFIX)) { identifier = identifier.substring(Group.IDENTIFIER_PREFIX.length()); Group roleGroup = ctxt.groups().getGroup(identifier); if (roleGroup != null) { ctxt.roles().save(new RoleAssignment(role.getRole(), - roleGroup, managedDv, privateUrlToken), false); + roleGroup, managedDv, privateUrlToken), false, getRequest()); } } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java index 5aa2ceb49e4..d30443c5de3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java @@ -126,7 +126,7 @@ protected void postDBFlush( Dataset theDataset, CommandContext ctxt ){ logger.info("New Dataset id: " + theDataset.getId()); RoleAssignment roleAssignment = new RoleAssignment(theDataset.getOwner().getDefaultContributorRole(), getRequest().getUser(), theDataset, privateUrlToken); - ctxt.roles().save(roleAssignment, false); + ctxt.roles().save(roleAssignment, false, getRequest()); // TODO: the above may be creating the role assignments and saving them // in the database, but without properly linking them to the dataset diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommandTest.java index 73880b78e7b..dd7d60632f9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommandTest.java @@ -101,15 +101,15 @@ public DataverseRole findBuiltinRoleByAlias(String alias) { } @Override - public RoleAssignment save(RoleAssignment assignment) { + public RoleAssignment save(RoleAssignment assignment, DataverseRequest req) { assignment.setId( nextId() ); assignments.add(assignment); return assignment; } @Override - public RoleAssignment save(RoleAssignment assignment, boolean index) { - return save (assignment); + public RoleAssignment save(RoleAssignment assignment, boolean index, DataverseRequest req) { + return save (assignment, req); } @Override diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommandTest.java index 0ba29f74774..e3ecf6b8a9a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommandTest.java @@ -11,6 +11,7 @@ import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser; import edu.harvard.iq.dataverse.engine.TestCommandContext; import edu.harvard.iq.dataverse.engine.TestDataverseEngine; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; @@ -79,7 +80,7 @@ public DataverseRole findBuiltinRoleByAlias(String alias) { } @Override - public RoleAssignment save(RoleAssignment assignment) { + public RoleAssignment save(RoleAssignment assignment, DataverseRequest req) { // no-op return assignment; } diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommandTest.java index 380a4bbcf18..b8902728785 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommandTest.java @@ -284,15 +284,15 @@ public DataverseRole findBuiltinRoleByAlias(String alias) { } @Override - public RoleAssignment save(RoleAssignment assignment) { + public RoleAssignment save(RoleAssignment assignment, DataverseRequest req) { assignment.setId(nextId()); assignments.add(assignment); return assignment; } @Override - public RoleAssignment save(RoleAssignment assignment, boolean index) { - return save(assignment); + public RoleAssignment save(RoleAssignment assignment, boolean index, DataverseRequest req) { + return save(assignment, req); } @Override diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommandTest.java index 68c44764dff..2c133f33932 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommandTest.java @@ -107,7 +107,7 @@ public DataverseRole findBuiltinRoleByAlias(String alias) { } @Override - public RoleAssignment save(RoleAssignment assignment) { + public RoleAssignment save(RoleAssignment assignment, DataverseRequest req) { // no-op return assignment; } diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/SubmitDatasetForReviewCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/SubmitDatasetForReviewCommandTest.java index 700ba332247..2f607aa7a0e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/SubmitDatasetForReviewCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/SubmitDatasetForReviewCommandTest.java @@ -113,7 +113,7 @@ public DataverseRole findBuiltinRoleByAlias(String alias) { } @Override - public RoleAssignment save(RoleAssignment assignment) { + public RoleAssignment save(RoleAssignment assignment, DataverseRequest req) { // no-op return assignment; } From d896af5d893bf7cdf8eb143c5d0530aad64f5c32 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 16 Apr 2025 14:41:14 -0400 Subject: [PATCH 025/634] update history when assignment made --- src/main/webapp/roles-assign.xhtml | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/main/webapp/roles-assign.xhtml b/src/main/webapp/roles-assign.xhtml index 93b9862c55d..8c5290b1580 100644 --- a/src/main/webapp/roles-assign.xhtml +++ b/src/main/webapp/roles-assign.xhtml @@ -89,7 +89,7 @@
@@ -100,20 +100,4 @@
- -

- #{bundle['dataverse.permissions.usersOrGroups.assignDialog.fileDownloadConfirm']} -

-
- - - - - -
- From 06042f3eaab71ca59cf22dc2deec2c5c7d17716c Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 16 Apr 2025 15:27:58 -0400 Subject: [PATCH 026/634] handle dataverse, no filePIDs, more update history --- .../harvard/iq/dataverse/RoleAssignmentAudit.java | 13 +++++++++---- src/main/webapp/permissions-manage-files.xhtml | 10 +++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java index dc6fb1806c2..52292aeb065 100644 --- a/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java +++ b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java @@ -61,16 +61,16 @@ public class RoleAssignmentAudit implements Serializable { @Column(name = "assignee_identifier", nullable = false) private String assigneeIdentifier; - @Column(name = "role_id") + @Column(name = "role_id", nullable = false) private Long roleId; @Column(name = "role_alias", nullable = false) private String roleAlias; - @Column(name = "definition_point_id") + @Column(name = "definition_point_id", nullable = false) private Long definitionPointId; - @Column(name = "definition_point_identifier", nullable = false) + @Column(name = "definition_point_identifier", nullable = true) private String definitionPointIdentifier; public enum ActionType { @@ -90,7 +90,12 @@ public RoleAssignmentAudit(RoleAssignment roleAssignment, DataverseRequest reque this.roleId = roleAssignment.getRole().getId(); this.roleAlias = roleAssignment.getRole().getAlias(); this.definitionPointId = roleAssignment.getDefinitionPoint().getId(); - this.definitionPointIdentifier = roleAssignment.getDefinitionPoint().getGlobalId().asString(); + GlobalId globalId = roleAssignment.getDefinitionPoint().getGlobalId(); + if(globalId != null) { + this.definitionPointIdentifier = roleAssignment.getDefinitionPoint().getGlobalId().asString(); + } else if(roleAssignment.getDefinitionPoint() instanceof Dataverse dv) { + this.definitionPointIdentifier = dv.getAlias(); + } } // Getters and setters diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index 9e43d4ce910..11fc97087fd 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -95,7 +95,7 @@ update=":#{p:resolveClientId('rolesPermissionsForm:userGroups', view)} :#{p:resolveClientId('rolesPermissionsForm:restrictedFiles', view)} #{p:resolveClientId('rolesPermissionsForm:usersGroups', view)} - @([id$=Messages])"> + roleAssignmentHistory @([id$=Messages])"> #{bundle['dataverse.permissionsFiles.assignDialog.grantBtn']} + roleAssignmentHistory @([id$=Messages])"> #{bundle['dataverse.permissionsFiles.assignDialog.rejectBtn']}
@@ -383,14 +383,14 @@
@@ -415,7 +415,7 @@
+
+ @@ -258,6 +260,7 @@ +
diff --git a/src/main/webapp/permissions-manage.xhtml b/src/main/webapp/permissions-manage.xhtml index dfab2a067a3..25b976f8e53 100644 --- a/src/main/webapp/permissions-manage.xhtml +++ b/src/main/webapp/permissions-manage.xhtml @@ -206,8 +206,10 @@ #{bundle['dataverse.permissions.history.description']}
+
- + + @@ -231,6 +233,7 @@ +
From 1864d0dad2e8a652bc07508fd31c0edcf6ff3269 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 18 Apr 2025 11:35:00 -0400 Subject: [PATCH 029/634] update sorting --- .../edu/harvard/iq/dataverse/ManageFilePermissionsPage.java | 5 ++++- .../java/edu/harvard/iq/dataverse/ManagePermissionsPage.java | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java index 78e092844c2..33314ef4fd3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java @@ -585,7 +585,10 @@ public List getRoleAssignmentHistory() { } roleAssignmentHistory.addAll(finalHistoryMap.values()); - roleAssignmentHistory.sort(Comparator.comparing(RoleAssignmentHistoryEntry::getAssignedAt).reversed()); + roleAssignmentHistory.sort(Comparator + .comparing(RoleAssignmentHistoryEntry::getRevokedAt, Comparator.nullsLast(Comparator.naturalOrder())) + .thenComparing(RoleAssignmentHistoryEntry::getAssignedAt, Comparator.nullsLast(Comparator.naturalOrder())) + .reversed()); } return roleAssignmentHistory; } diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index ff6fd659703..ff638a88866 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -270,7 +270,7 @@ public List getRoleAssignmentHistory() { roleAssignmentHistory.addAll(historyMap.values()); roleAssignmentHistory.sort(Comparator - .comparing(RoleAssignmentHistoryEntry::getRevokedAt, Comparator.nullsFirst(Comparator.naturalOrder())) + .comparing(RoleAssignmentHistoryEntry::getRevokedAt, Comparator.nullsLast(Comparator.naturalOrder())) .thenComparing(RoleAssignmentHistoryEntry::getAssignedAt, Comparator.nullsLast(Comparator.naturalOrder())) .reversed()); }; From 65dc679465cc2d6ba367ca1473d307c105921cdd Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 18 Apr 2025 11:43:41 -0400 Subject: [PATCH 030/634] sortable, deferred load --- .../webapp/permissions-manage-files.xhtml | 78 ++++++++++--------- src/main/webapp/permissions-manage.xhtml | 55 +++++++------ 2 files changed, 72 insertions(+), 61 deletions(-) diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index 516ee9da0a1..4d8612cae65 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -219,6 +219,7 @@ +
@@ -226,42 +227,47 @@ #{bundle['dataverse.permissions.history.description']}
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
diff --git a/src/main/webapp/permissions-manage.xhtml b/src/main/webapp/permissions-manage.xhtml index 25b976f8e53..c47d4f773a6 100644 --- a/src/main/webapp/permissions-manage.xhtml +++ b/src/main/webapp/permissions-manage.xhtml @@ -208,31 +208,36 @@
- - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + +
From c87220ea6a4e510999a3a3c390b415ec9fa1d151 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 18 Apr 2025 13:05:06 -0400 Subject: [PATCH 031/634] cleanup/drop deferred load (wasn't working) --- .../webapp/permissions-manage-files.xhtml | 85 +++++++++---------- src/main/webapp/permissions-manage.xhtml | 72 ++++++++-------- 2 files changed, 72 insertions(+), 85 deletions(-) diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index 4d8612cae65..b5853783ef9 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -219,56 +219,47 @@ -
-
- #{bundle['dataverse.permissions.history']} - #{bundle['dataverse.permissions.history.description']} -
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- +
+ #{bundle['dataverse.permissions.history']} #{bundle['dataverse.permissions.history.description']} +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
diff --git a/src/main/webapp/permissions-manage.xhtml b/src/main/webapp/permissions-manage.xhtml index c47d4f773a6..d44b0bb47cc 100644 --- a/src/main/webapp/permissions-manage.xhtml +++ b/src/main/webapp/permissions-manage.xhtml @@ -201,47 +201,43 @@
-
- #{bundle['dataverse.permissions.history']} - #{bundle['dataverse.permissions.history.description']} -
-
- +
+ #{bundle['dataverse.permissions.history']} #{bundle['dataverse.permissions.history.description']} +
+
+
- - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + +
-
+
+
From cb57117b67aa49f6732e0f91eba4d62712124552 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 28 Apr 2025 13:53:30 -0400 Subject: [PATCH 032/634] make updating RA history conditional on flag being set. --- src/main/webapp/permissions-manage-files.xhtml | 16 +++++++++------- src/main/webapp/permissions-manage.xhtml | 8 +++++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index b5853783ef9..c015de4f5b3 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -23,6 +23,8 @@ + +
@@ -95,7 +97,7 @@ update=":#{p:resolveClientId('rolesPermissionsForm:userGroups', view)} :#{p:resolveClientId('rolesPermissionsForm:restrictedFiles', view)} #{p:resolveClientId('rolesPermissionsForm:usersGroups', view)} - roleAssignmentHistory @([id$=Messages])"> + @if(#{RAAuditingEnabled}) roleAssignmentHistory @([id$=Messages])"> #{bundle['dataverse.permissionsFiles.assignDialog.grantBtn']} + @if(#{RAAuditingEnabled}) roleAssignmentHistory @([id$=Messages])"> #{bundle['dataverse.permissionsFiles.assignDialog.rejectBtn']}
@@ -219,8 +221,8 @@ - -
+ +
#{bundle['dataverse.permissions.history']} #{bundle['dataverse.permissions.history.description']}
@@ -383,14 +385,14 @@
@@ -415,7 +417,7 @@ From 04b37f25de54741457fccc974b6ab9fca6b57b93 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 28 Apr 2025 14:08:32 -0400 Subject: [PATCH 033/634] missing widgetVar --- src/main/webapp/permissions-manage-files.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index c015de4f5b3..884925e1b98 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -228,7 +228,7 @@
- From b6b86ea88aae9f09ba60effc1d4e031098b8d0d0 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 28 Apr 2025 14:55:40 -0400 Subject: [PATCH 034/634] Revert "make updating RA history conditional on flag being set." This reverts commit 8f58f193482811c1279e3721f3971aaf3ce07dbe. --- src/main/webapp/permissions-manage-files.xhtml | 16 +++++++--------- src/main/webapp/permissions-manage.xhtml | 8 +++----- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index 884925e1b98..282e1da9b17 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -23,8 +23,6 @@ - -
@@ -97,7 +95,7 @@ update=":#{p:resolveClientId('rolesPermissionsForm:userGroups', view)} :#{p:resolveClientId('rolesPermissionsForm:restrictedFiles', view)} #{p:resolveClientId('rolesPermissionsForm:usersGroups', view)} - @if(#{RAAuditingEnabled}) roleAssignmentHistory @([id$=Messages])"> + roleAssignmentHistory @([id$=Messages])"> #{bundle['dataverse.permissionsFiles.assignDialog.grantBtn']} + roleAssignmentHistory @([id$=Messages])"> #{bundle['dataverse.permissionsFiles.assignDialog.rejectBtn']}
@@ -221,8 +219,8 @@
- -
+ +
#{bundle['dataverse.permissions.history']} #{bundle['dataverse.permissions.history.description']}
@@ -385,14 +383,14 @@
@@ -417,7 +415,7 @@ From d3ab76bf39845aa02b96135c8bd8a80687d80fec Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 28 Apr 2025 16:12:05 -0400 Subject: [PATCH 035/634] update to fix 500 error re not finding ra history --- src/main/webapp/permissions-manage-files.xhtml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index 282e1da9b17..1a93527dfef 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -95,7 +95,8 @@ update=":#{p:resolveClientId('rolesPermissionsForm:userGroups', view)} :#{p:resolveClientId('rolesPermissionsForm:restrictedFiles', view)} #{p:resolveClientId('rolesPermissionsForm:usersGroups', view)} - roleAssignmentHistory @([id$=Messages])"> + #{p:resolveClientId('rolesPermissionsForm:roleAssignmentHistory', view)} + @([id$=Messages])"> #{bundle['dataverse.permissionsFiles.assignDialog.grantBtn']} + #{p:resolveClientId('rolesPermissionsForm:roleAssignmentHistory', view)} + @([id$=Messages])"> #{bundle['dataverse.permissionsFiles.assignDialog.rejectBtn']}
From af6600f1a26483ad0cbf2d426ff9d9cc10b218fe Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 29 Apr 2025 08:56:53 -0400 Subject: [PATCH 036/634] word wrapping on perms form --- src/main/webapp/resources/css/structure.css | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index 4d83ddd489d..9e12a83f4a0 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -568,6 +568,7 @@ div.ui-tabs-panels .ui-tabs.ui-tabs-top .ui-tabs-nav.ui-widget-header {border-bo /* -------- PERMISSIONS -------- */ div[id$='roleDisplay'] span.label, div[id$='roleDetails'] span.label {display:inline-block; margin-bottom:4px;} +#rolesPermissionsForm {word-break:break-word;} /* -------- METRICS -------- */ /* -- OLD LAYOUT, DATAVERSE -- */ From 4fca21b37d867aa20f31933c0c38144a3d85ca5f Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 7 Jul 2025 14:32:00 -0400 Subject: [PATCH 037/634] missing import --- .../java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java index 09097be9c8e..a6e2a53dadd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java @@ -14,6 +14,7 @@ import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.logging.Logger; import java.util.stream.Collectors; From 8c0bfbf706b94cacfdc7a1fc803c0106af85203b Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 7 Jul 2025 17:22:12 -0400 Subject: [PATCH 038/634] refactor getting RA history --- .../dataverse/DataverseRoleServiceBean.java | 162 ++++++++++++++++++ .../dataverse/ManageFilePermissionsPage.java | 48 +----- .../iq/dataverse/ManagePermissionsPage.java | 111 +----------- 3 files changed, 169 insertions(+), 152 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java index a6e2a53dadd..68fee4aacad 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse; +import edu.harvard.iq.dataverse.DataverseRoleServiceBean.RoleAssignmentHistoryEntry; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.RoleAssignee; @@ -12,9 +13,13 @@ import edu.harvard.iq.dataverse.search.SolrIndexServiceBean; import java.util.ArrayList; import java.util.Collection; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Map; import java.util.Set; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -367,5 +372,162 @@ For a given permission and dataverse Id get all of the roles (built-in or owned } return retVal; } + + /** + * Retrieves role assignment history for a specific definition point + * + * @param definitionPointId The ID of the definition point + * @return List of role assignment history entries + */ + public List getRoleAssignmentHistory(Long definitionPointId) { + List audits = em.createNamedQuery("RoleAssignmentAudit.findByDefinitionPointId", RoleAssignmentAudit.class) + .setParameter("definitionPointId", definitionPointId) + .getResultList(); + + return processRoleAssignmentAudits(audits, false); + } + + /** + * Retrieves role assignment history for all files in a dataset + * + * @param datasetId The ID of the dataset + * @return List of role assignment history entries + */ + public List getChildRoleAssignmentHistory(Long datasetId) { + List audits = em.createNamedQuery("RoleAssignmentAudit.findByOwnerId", RoleAssignmentAudit.class) + .setParameter("datasetId", datasetId) + .getResultList(); + + return processRoleAssignmentAudits(audits, true); + } + + /** + * Common method to process role assignment audits and create history entries + * + * @param audits List of role assignment audit records + * @param combineEntries Whether to combine entries for different files + * @return List of role assignment history entries + */ + private List processRoleAssignmentAudits(List audits, boolean combineEntries) { + List roleAssignmentHistory = new ArrayList<>(); + Map historyMap = new HashMap<>(); + + // First pass: Create entries from audit records + for (RoleAssignmentAudit audit : audits) { + Long roleAssignmentId = audit.getRoleAssignmentId(); + RoleAssignmentHistoryEntry entry = historyMap.get(roleAssignmentId); + + if (entry == null) { + entry = new RoleAssignmentHistoryEntry(audit.getAssigneeIdentifier(), audit.getRoleAlias(), audit.getDefinitionPointId()); + historyMap.put(roleAssignmentId, entry); + } + + if (audit.getActionType() == RoleAssignmentAudit.ActionType.ASSIGN) { + entry.setAssignedBy(audit.getActionByIdentifier()); + entry.setAssignedAt(audit.getActionTimestamp()); + } else if (audit.getActionType() == RoleAssignmentAudit.ActionType.REVOKE) { + entry.setRevokedBy(audit.getActionByIdentifier()); + entry.setRevokedAt(audit.getActionTimestamp()); + } + } + + // Second pass: Combine entries with matching criteria if requested + if (combineEntries) { + Map finalHistoryMap = new HashMap<>(); + for (RoleAssignmentHistoryEntry entry : historyMap.values()) { + String key = entry.getAssigneeIdentifier() + "|" + entry.getRoleName() + "|" + + entry.getAssignedBy() + "|" + entry.getAssignedAt() + "|" + + entry.getRevokedBy() + "|" + entry.getRevokedAt(); + + RoleAssignmentHistoryEntry existingEntry = finalHistoryMap.get(key); + if (existingEntry == null) { + finalHistoryMap.put(key, entry); + } else { + existingEntry.addDefinitionPointId(entry.getDefinitionPointIds().get(0)); + } + } + roleAssignmentHistory.addAll(finalHistoryMap.values()); + } else { + roleAssignmentHistory.addAll(historyMap.values()); + } + + // Sort the entries + roleAssignmentHistory.sort(Comparator + .comparing(RoleAssignmentHistoryEntry::getRevokedAt, Comparator.nullsLast(Comparator.naturalOrder())) + .thenComparing(RoleAssignmentHistoryEntry::getAssignedAt, Comparator.nullsLast(Comparator.naturalOrder())) + .reversed()); + return roleAssignmentHistory; + } + + public static class RoleAssignmentHistoryEntry { + private String roleName; + private String assigneeIdentifier; + private String assignedBy; + private Date assignedAt; + private String revokedBy; + private Date revokedAt; + private List definitionPointIds; // New field + + public RoleAssignmentHistoryEntry(String assigneeIdentifier, String roleName, Long definitionPointId) { + this.roleName = roleName; + this.assigneeIdentifier = assigneeIdentifier; + this.definitionPointIds = new ArrayList(); + definitionPointIds.add(definitionPointId); + } + + public void setRevokedAt(Date actionTimestamp) { + revokedAt = actionTimestamp; + } + + public void setRevokedBy(String actionByIdentifier) { + revokedBy = actionByIdentifier; + } + + public void setAssignedAt(Date actionTimestamp) { + assignedAt = actionTimestamp; + } + + public void setAssignedBy(String actionByIdentifier) { + assignedBy = actionByIdentifier; + } + + public String getRoleName() { + return roleName; + } + + public String getAssigneeIdentifier() { + return assigneeIdentifier; + } + + public String getAssignedBy() { + return assignedBy; + } + + public Date getAssignedAt() { + return assignedAt; + } + + public String getRevokedBy() { + return revokedBy; + } + + public Date getRevokedAt() { + return revokedAt; + } + + public List getDefinitionPointIds() { + return definitionPointIds; + } + + public void addDefinitionPointId(Long definitionPointId) { + definitionPointIds.add(definitionPointId); + } + + public String getDefinitionPointIdsAsString() { + return definitionPointIds.stream() + .map(Object::toString) + .collect(Collectors.joining(", ")); + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java index 33314ef4fd3..1cd59bbdf57 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java @@ -5,7 +5,7 @@ */ package edu.harvard.iq.dataverse; -import edu.harvard.iq.dataverse.ManagePermissionsPage.RoleAssignmentHistoryEntry; +import edu.harvard.iq.dataverse.DataverseRoleServiceBean.RoleAssignmentHistoryEntry; import edu.harvard.iq.dataverse.api.Util; import edu.harvard.iq.dataverse.authorization.AuthenticationProvider; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; @@ -544,51 +544,7 @@ private boolean assignRole(RoleAssignee ra, DataFile file, DataverseRole r) { public List getRoleAssignmentHistory() { if (roleAssignmentHistory == null) { - roleAssignmentHistory = new ArrayList<>(); - - List audits = em.createNamedQuery("RoleAssignmentAudit.findByOwnerId", RoleAssignmentAudit.class) - .setParameter("datasetId", dataset.getId()) - .getResultList(); - - Map historyMap = new HashMap<>(); - - for (RoleAssignmentAudit audit : audits) { - Long roleAssignmentId = audit.getRoleAssignmentId(); - RoleAssignmentHistoryEntry entry = historyMap.get(roleAssignmentId); - - if (entry == null) { - entry = new RoleAssignmentHistoryEntry(audit.getAssigneeIdentifier(), audit.getRoleAlias(), audit.getDefinitionPointId() ); - historyMap.put(roleAssignmentId, entry); - } - - if (audit.getActionType() == RoleAssignmentAudit.ActionType.ASSIGN) { - entry.setAssignedBy(audit.getActionByIdentifier()); - entry.setAssignedAt(audit.getActionTimestamp()); - } else if (audit.getActionType() == RoleAssignmentAudit.ActionType.REVOKE) { - entry.setRevokedBy(audit.getActionByIdentifier()); - entry.setRevokedAt(audit.getActionTimestamp()); - } - } - // Second pass: Combine entries with matching criteria - Map finalHistoryMap = new HashMap<>(); - for (RoleAssignmentHistoryEntry entry : historyMap.values()) { - String key = entry.getAssigneeIdentifier() + "|" + entry.getRoleName() + "|" + - entry.getAssignedBy() + "|" + entry.getAssignedAt() + "|" + - entry.getRevokedBy() + "|" + entry.getRevokedAt(); - - RoleAssignmentHistoryEntry existingEntry = finalHistoryMap.get(key); - if (existingEntry == null) { - finalHistoryMap.put(key, entry); - } else { - existingEntry.addDefinitionPointId(entry.getDefinitionPointIds().get(0)); - } - } - - roleAssignmentHistory.addAll(finalHistoryMap.values()); - roleAssignmentHistory.sort(Comparator - .comparing(RoleAssignmentHistoryEntry::getRevokedAt, Comparator.nullsLast(Comparator.naturalOrder())) - .thenComparing(RoleAssignmentHistoryEntry::getAssignedAt, Comparator.nullsLast(Comparator.naturalOrder())) - .reversed()); + roleAssignmentHistory = roleService.getChildRoleAssignmentHistory(dataset.getId()); } return roleAssignmentHistory; } diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index ff638a88866..9401f112d51 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -35,7 +35,6 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; import jakarta.ejb.EJB; import jakarta.faces.application.FacesMessage; @@ -238,42 +237,13 @@ public void editRole(String roleId) { } /** Role Assignment History */ - private List roleAssignmentHistory; + private List roleAssignmentHistory; - public List getRoleAssignmentHistory() { + public List getRoleAssignmentHistory() { + if (roleAssignmentHistory == null) { - roleAssignmentHistory = new ArrayList<>(); - - List audits = em.createNamedQuery("RoleAssignmentAudit.findByDefinitionPointId", RoleAssignmentAudit.class) - .setParameter("definitionPointId", dvObject.getId()) - .getResultList(); - - Map historyMap = new HashMap<>(); - - for (RoleAssignmentAudit audit : audits) { - Long roleAssignmentId = audit.getRoleAssignmentId(); - RoleAssignmentHistoryEntry entry = historyMap.get(roleAssignmentId); - - if (entry == null) { - entry = new RoleAssignmentHistoryEntry(audit.getAssigneeIdentifier(), audit.getRoleAlias(), audit.getDefinitionPointId()); - historyMap.put(roleAssignmentId, entry); - } - - if (audit.getActionType() == RoleAssignmentAudit.ActionType.ASSIGN) { - entry.setAssignedBy(audit.getActionByIdentifier()); - entry.setAssignedAt(audit.getActionTimestamp()); - } else if (audit.getActionType() == RoleAssignmentAudit.ActionType.REVOKE) { - entry.setRevokedBy(audit.getActionByIdentifier()); - entry.setRevokedAt(audit.getActionTimestamp()); - } - } - - roleAssignmentHistory.addAll(historyMap.values()); - roleAssignmentHistory.sort(Comparator - .comparing(RoleAssignmentHistoryEntry::getRevokedAt, Comparator.nullsLast(Comparator.naturalOrder())) - .thenComparing(RoleAssignmentHistoryEntry::getAssignedAt, Comparator.nullsLast(Comparator.naturalOrder())) - .reversed()); - }; + roleAssignmentHistory = roleService.getRoleAssignmentHistory(dvObject.getId()); + } return roleAssignmentHistory; } @@ -763,75 +733,4 @@ public Long getId() { } } - - public static class RoleAssignmentHistoryEntry { - private String roleName; - private String assigneeIdentifier; - private String assignedBy; - private Date assignedAt; - private String revokedBy; - private Date revokedAt; - private List definitionPointIds; // New field - - public RoleAssignmentHistoryEntry(String assigneeIdentifier, String roleName, Long definitionPointId) { - this.roleName = roleName; - this.assigneeIdentifier = assigneeIdentifier; - this.definitionPointIds = new ArrayList(); - definitionPointIds.add(definitionPointId); - } - - public void setRevokedAt(Date actionTimestamp) { - revokedAt = actionTimestamp; - } - - public void setRevokedBy(String actionByIdentifier) { - revokedBy = actionByIdentifier; - } - - public void setAssignedAt(Date actionTimestamp) { - assignedAt = actionTimestamp; - } - - public void setAssignedBy(String actionByIdentifier) { - assignedBy = actionByIdentifier; - } - - public String getRoleName() { - return roleName; - } - - public String getAssigneeIdentifier() { - return assigneeIdentifier; - } - - public String getAssignedBy() { - return assignedBy; - } - - public Date getAssignedAt() { - return assignedAt; - } - - public String getRevokedBy() { - return revokedBy; - } - - public Date getRevokedAt() { - return revokedAt; - } - - public List getDefinitionPointIds() { - return definitionPointIds; - } - - public void addDefinitionPointId(Long definitionPointId) { - definitionPointIds.add(definitionPointId); - } - - public String getDefinitionPointIdsAsString() { - return definitionPointIds.stream() - .map(Object::toString) - .collect(Collectors.joining(", ")); - } - } } From 859d0ab515f97eb449adad1f37bdb2daf1b3f067 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 7 Jul 2025 17:22:20 -0400 Subject: [PATCH 039/634] api call --- .../harvard/iq/dataverse/api/Datasets.java | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 878ebde0504..b818595400a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3,6 +3,7 @@ import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.DatasetLock.Reason; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; +import edu.harvard.iq.dataverse.DataverseRoleServiceBean.RoleAssignmentHistoryEntry; import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.api.dto.RoleAssignmentDTO; @@ -5988,5 +5989,59 @@ public Response deleteVersionNote(@Context ContainerRequestContext crc, @PathPar return ok("Note deleted"); }, getRequestUser(crc)); } + + @GET + @AuthRequired + @Path("{identifier}/permissions/history") + public Response getRoleAssignmentHistory(@Context ContainerRequestContext crc, @PathParam("identifier") String id) { + return response(req -> { + Dataset dataset = findDatasetOrDie(id); + + // user is authenticated + AuthenticatedUser authenticatedUser = null; + try { + authenticatedUser = getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return error(Status.UNAUTHORIZED, "Authentication is required."); + } + + // Check if the user has permission to manage permissions for this dataset + if (!permissionService.userOn(authenticatedUser, dataset).has(Permission.ManageDatasetPermissions)) { + return error(Status.FORBIDDEN, "You do not have permission to view the role assignment history for this dataset"); + } + + // Get the role assignment history + ManagePermissionsPage managePermissionsPage = new ManagePermissionsPage(); + managePermissionsPage.setDvObject(dataset); + List history = managePermissionsPage.getRoleAssignmentHistory(); + + // Convert to JSON array + JsonArrayBuilder jsonArray = Json.createArrayBuilder(); + for (DataverseRoleServiceBean.RoleAssignmentHistoryEntry entry : history) { + JsonObjectBuilder job = Json.createObjectBuilder() + .add("assigneeIdentifier", entry.getAssigneeIdentifier()) + .add("roleName", entry.getRoleName()) + .add("assignedBy", entry.getAssignedBy()) + .add("assignedAt", entry.getAssignedAt().toString()); + + // Add revocation info if available + if (entry.getRevokedBy() != null) { + job.add("revokedBy", entry.getRevokedBy()); + } else { + job.add("revokedBy", JsonValue.NULL); + } + + if (entry.getRevokedAt() != null) { + job.add("revokedAt", entry.getRevokedAt().toString()); + } else { + job.add("revokedAt", JsonValue.NULL); + } + + jsonArray.add(job); + } + + return ok(jsonArray); + }, getRequestUser(crc)); + } } From 1c87cef8bfbcd437fde6454d20ba75c7665d7e1f Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 16 Jul 2025 17:07:42 -0400 Subject: [PATCH 040/634] remove unused imports --- .../java/edu/harvard/iq/dataverse/ManagePermissionsPage.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index 9401f112d51..4fbf520154b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -26,12 +26,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.Date; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; From a6a1ab1890e2505977088ca067554b838cc6d014 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 16 Jul 2025 17:07:54 -0400 Subject: [PATCH 041/634] api call with i18n and csv option --- .../harvard/iq/dataverse/api/Datasets.java | 76 ++++++++++++++----- 1 file changed, 59 insertions(+), 17 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index b818595400a..67f6e552057 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -5993,10 +5993,13 @@ public Response deleteVersionNote(@Context ContainerRequestContext crc, @PathPar @GET @AuthRequired @Path("{identifier}/permissions/history") - public Response getRoleAssignmentHistory(@Context ContainerRequestContext crc, @PathParam("identifier") String id) { + @Produces({ MediaType.APPLICATION_JSON, "text/csv" }) + public Response getRoleAssignmentHistory(@Context ContainerRequestContext crc, + @PathParam("identifier") String id, + @Context HttpHeaders headers) { return response(req -> { Dataset dataset = findDatasetOrDie(id); - + // user is authenticated AuthenticatedUser authenticatedUser = null; try { @@ -6004,42 +6007,81 @@ public Response getRoleAssignmentHistory(@Context ContainerRequestContext crc, @ } catch (WrappedResponse ex) { return error(Status.UNAUTHORIZED, "Authentication is required."); } - + // Check if the user has permission to manage permissions for this dataset if (!permissionService.userOn(authenticatedUser, dataset).has(Permission.ManageDatasetPermissions)) { return error(Status.FORBIDDEN, "You do not have permission to view the role assignment history for this dataset"); } - + // Get the role assignment history - ManagePermissionsPage managePermissionsPage = new ManagePermissionsPage(); - managePermissionsPage.setDvObject(dataset); - List history = managePermissionsPage.getRoleAssignmentHistory(); - - // Convert to JSON array + List history = dataverseRoleService.getRoleAssignmentHistory(dataset.getId()); + + List acceptedTypes = headers.getAcceptableMediaTypes(); + boolean wantCSV = acceptedTypes.stream() + .anyMatch(mt -> mt.toString().equals("text/csv")); + + if (wantCSV) { + String assigneeHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.assignee"); + String roleHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.role"); + String assignedByHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.assignedBy"); + String assignedAtHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.assignedAt"); + String revokedByHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.revokedBy"); + String revokedAtHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.revokedAt"); + + // Generate CSV response + StringBuilder csvBuilder = new StringBuilder(); + // Add CSV header with internationalized column names + csvBuilder.append(assigneeHeader).append(",") + .append(roleHeader).append(",") + .append(assignedByHeader).append(",") + .append(assignedAtHeader).append(",") + .append(revokedByHeader).append(",") + .append(revokedAtHeader).append("\n"); + + // Add data rows + for (DataverseRoleServiceBean.RoleAssignmentHistoryEntry entry : history) { + csvBuilder.append(entry.getAssigneeIdentifier()).append(","); + csvBuilder.append(entry.getRoleName()).append(","); + csvBuilder.append(entry.getAssignedBy()).append(","); + csvBuilder.append(entry.getAssignedAt().toString()).append(","); + + // Handle nullable fields + csvBuilder.append(entry.getRevokedBy() != null ? entry.getRevokedBy() : "").append(","); + csvBuilder.append(entry.getRevokedAt() != null ? entry.getRevokedAt().toString() : ""); + csvBuilder.append("\n"); + } + + return Response.ok() + .entity(csvBuilder.toString()) + .type("text/csv") + .header("Content-Disposition", "attachment; filename=permissions_history.csv") + .build(); + } + // Or Json by default JsonArrayBuilder jsonArray = Json.createArrayBuilder(); for (DataverseRoleServiceBean.RoleAssignmentHistoryEntry entry : history) { JsonObjectBuilder job = Json.createObjectBuilder() - .add("assigneeIdentifier", entry.getAssigneeIdentifier()) - .add("roleName", entry.getRoleName()) - .add("assignedBy", entry.getAssignedBy()) - .add("assignedAt", entry.getAssignedAt().toString()); - + .add("assigneeIdentifier", entry.getAssigneeIdentifier()) + .add("roleName", entry.getRoleName()) + .add("assignedBy", entry.getAssignedBy()) + .add("assignedAt", entry.getAssignedAt().toString()); + // Add revocation info if available if (entry.getRevokedBy() != null) { job.add("revokedBy", entry.getRevokedBy()); } else { job.add("revokedBy", JsonValue.NULL); } - + if (entry.getRevokedAt() != null) { job.add("revokedAt", entry.getRevokedAt().toString()); } else { job.add("revokedAt", JsonValue.NULL); } - + jsonArray.add(job); } - + return ok(jsonArray); }, getRequestUser(crc)); } From c2c54dce9229fee7a4b5cb2db44528fd411d7dbf Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 16 Jul 2025 17:08:23 -0400 Subject: [PATCH 042/634] dataset/dataverse button to download csv --- src/main/webapp/permissions-manage.xhtml | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/main/webapp/permissions-manage.xhtml b/src/main/webapp/permissions-manage.xhtml index d44b0bb47cc..ef9d9345b32 100644 --- a/src/main/webapp/permissions-manage.xhtml +++ b/src/main/webapp/permissions-manage.xhtml @@ -207,6 +207,41 @@
+
+ + #{bundle['dataverse.permissions.history.download']} + + + #{bundle['dataverse.permissions.history.download']} + +
+ + Date: Thu, 17 Jul 2025 13:46:32 -0400 Subject: [PATCH 043/634] refactor, apis for all three types --- .../iq/dataverse/api/AbstractApiBean.java | 90 +++++++++++++++++++ .../harvard/iq/dataverse/api/Datasets.java | 76 +--------------- .../harvard/iq/dataverse/api/Dataverses.java | 23 +++++ .../edu/harvard/iq/dataverse/api/Files.java | 23 +++++ 4 files changed, 137 insertions(+), 75 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 018657bff4d..a6562b7c1d9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -5,6 +5,7 @@ import static edu.harvard.iq.dataverse.api.Datasets.handleVersion; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.DataverseRole; +import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.RoleAssignee; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; @@ -45,6 +46,7 @@ import jakarta.validation.constraints.NotNull; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.HttpHeaders; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.Response.ResponseBuilder; @@ -643,6 +645,94 @@ protected DatasetFieldType findDatasetFieldType(String idtf) throws NumberFormat return isNumeric(idtf) ? datasetFieldSvc.find(Long.parseLong(idtf)) : datasetFieldSvc.findByNameOpt(idtf); } + + /** + * Gets role assignment history for a DvObject (Dataset, Dataverse, or DataFile) + * + * @param dvObject The DvObject to get history for + * @param authenticatedUser The authenticated user making the request + * @param headers HTTP headers from the request (for content negotiation) + * @return Response containing history in JSON or CSV format + */ + protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, AuthenticatedUser authenticatedUser, HttpHeaders headers) { + // Check if the user has permission to manage permissions for this object + if (!permissionSvc.userOn(authenticatedUser, dvObject).has(Permission.ManageDatasetPermissions)) { + return error(Status.FORBIDDEN, "You do not have permission to view the role assignment history for this " + dvObject.getClass().getSimpleName().toLowerCase()); + } + + // Get the role assignment history + List history = rolesSvc.getRoleAssignmentHistory(dvObject.getId()); + + List acceptedTypes = headers.getAcceptableMediaTypes(); + boolean wantCSV = acceptedTypes.stream() + .anyMatch(mt -> mt.toString().equals("text/csv")); + + if (wantCSV) { + String assigneeHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.assignee"); + String roleHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.role"); + String assignedByHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.assignedBy"); + String assignedAtHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.assignedAt"); + String revokedByHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.revokedBy"); + String revokedAtHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.revokedAt"); + + // Generate CSV response + StringBuilder csvBuilder = new StringBuilder(); + // Add CSV header with internationalized column names + csvBuilder.append(assigneeHeader).append(",") + .append(roleHeader).append(",") + .append(assignedByHeader).append(",") + .append(assignedAtHeader).append(",") + .append(revokedByHeader).append(",") + .append(revokedAtHeader).append("\n"); + + // Add data rows + for (DataverseRoleServiceBean.RoleAssignmentHistoryEntry entry : history) { + csvBuilder.append(entry.getAssigneeIdentifier()).append(","); + csvBuilder.append(entry.getRoleName()).append(","); + csvBuilder.append(entry.getAssignedBy()).append(","); + csvBuilder.append(entry.getAssignedAt().toString()).append(","); + + // Handle nullable fields + csvBuilder.append(entry.getRevokedBy() != null ? entry.getRevokedBy() : "").append(","); + csvBuilder.append(entry.getRevokedAt() != null ? entry.getRevokedAt().toString() : ""); + csvBuilder.append("\n"); + } + + String objectType = dvObject.getClass().getSimpleName().toLowerCase(); + return Response.ok() + .entity(csvBuilder.toString()) + .type("text/csv") + .header("Content-Disposition", "attachment; filename=" + objectType + "_permissions_history.csv") + .build(); + } + + // Or Json by default + JsonArrayBuilder jsonArray = Json.createArrayBuilder(); + for (DataverseRoleServiceBean.RoleAssignmentHistoryEntry entry : history) { + JsonObjectBuilder job = Json.createObjectBuilder() + .add("assigneeIdentifier", entry.getAssigneeIdentifier()) + .add("roleName", entry.getRoleName()) + .add("assignedBy", entry.getAssignedBy()) + .add("assignedAt", entry.getAssignedAt().toString()); + + // Add revocation info if available + if (entry.getRevokedBy() != null) { + job.add("revokedBy", entry.getRevokedBy()); + } else { + job.add("revokedBy", JsonValue.NULL); + } + + if (entry.getRevokedAt() != null) { + job.add("revokedAt", entry.getRevokedAt().toString()); + } else { + job.add("revokedAt", JsonValue.NULL); + } + + jsonArray.add(job); + } + + return ok(jsonArray); + } /* =================== *\ * Command Execution * diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 67f6e552057..9bbf191dda1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -6008,81 +6008,7 @@ public Response getRoleAssignmentHistory(@Context ContainerRequestContext crc, return error(Status.UNAUTHORIZED, "Authentication is required."); } - // Check if the user has permission to manage permissions for this dataset - if (!permissionService.userOn(authenticatedUser, dataset).has(Permission.ManageDatasetPermissions)) { - return error(Status.FORBIDDEN, "You do not have permission to view the role assignment history for this dataset"); - } - - // Get the role assignment history - List history = dataverseRoleService.getRoleAssignmentHistory(dataset.getId()); - - List acceptedTypes = headers.getAcceptableMediaTypes(); - boolean wantCSV = acceptedTypes.stream() - .anyMatch(mt -> mt.toString().equals("text/csv")); - - if (wantCSV) { - String assigneeHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.assignee"); - String roleHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.role"); - String assignedByHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.assignedBy"); - String assignedAtHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.assignedAt"); - String revokedByHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.revokedBy"); - String revokedAtHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.revokedAt"); - - // Generate CSV response - StringBuilder csvBuilder = new StringBuilder(); - // Add CSV header with internationalized column names - csvBuilder.append(assigneeHeader).append(",") - .append(roleHeader).append(",") - .append(assignedByHeader).append(",") - .append(assignedAtHeader).append(",") - .append(revokedByHeader).append(",") - .append(revokedAtHeader).append("\n"); - - // Add data rows - for (DataverseRoleServiceBean.RoleAssignmentHistoryEntry entry : history) { - csvBuilder.append(entry.getAssigneeIdentifier()).append(","); - csvBuilder.append(entry.getRoleName()).append(","); - csvBuilder.append(entry.getAssignedBy()).append(","); - csvBuilder.append(entry.getAssignedAt().toString()).append(","); - - // Handle nullable fields - csvBuilder.append(entry.getRevokedBy() != null ? entry.getRevokedBy() : "").append(","); - csvBuilder.append(entry.getRevokedAt() != null ? entry.getRevokedAt().toString() : ""); - csvBuilder.append("\n"); - } - - return Response.ok() - .entity(csvBuilder.toString()) - .type("text/csv") - .header("Content-Disposition", "attachment; filename=permissions_history.csv") - .build(); - } - // Or Json by default - JsonArrayBuilder jsonArray = Json.createArrayBuilder(); - for (DataverseRoleServiceBean.RoleAssignmentHistoryEntry entry : history) { - JsonObjectBuilder job = Json.createObjectBuilder() - .add("assigneeIdentifier", entry.getAssigneeIdentifier()) - .add("roleName", entry.getRoleName()) - .add("assignedBy", entry.getAssignedBy()) - .add("assignedAt", entry.getAssignedAt().toString()); - - // Add revocation info if available - if (entry.getRevokedBy() != null) { - job.add("revokedBy", entry.getRevokedBy()); - } else { - job.add("revokedBy", JsonValue.NULL); - } - - if (entry.getRevokedAt() != null) { - job.add("revokedAt", entry.getRevokedAt().toString()); - } else { - job.add("revokedAt", JsonValue.NULL); - } - - jsonArray.add(job); - } - - return ok(jsonArray); + return getRoleAssignmentHistoryResponse(dataset, authenticatedUser, headers); }, getRequestUser(crc)); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index b1d56b6b8a9..cc850dee1f4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -71,6 +71,7 @@ import jakarta.servlet.http.HttpServletResponse; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.HttpHeaders; import jakarta.ws.rs.core.StreamingOutput; import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @@ -1931,4 +1932,26 @@ public Response deleteFeaturedItems(@Context ContainerRequestContext crc, @PathP return e.getResponse(); } } + + @GET + @AuthRequired + @Path("{identifier}/permissions/history") + @Produces({ MediaType.APPLICATION_JSON, "text/csv" }) + public Response getRoleAssignmentHistory(@Context ContainerRequestContext crc, + @PathParam("identifier") String id, + @Context HttpHeaders headers) { + return response(req -> { + Dataverse dataverse = findDataverseOrDie(id); + + // user is authenticated + AuthenticatedUser authenticatedUser = null; + try { + authenticatedUser = getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return error(Status.UNAUTHORIZED, "Authentication is required."); + } + + return getRoleAssignmentHistoryResponse(dataverse, authenticatedUser, headers); + }, getRequestUser(crc)); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 61a69236f57..59470530047 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -60,6 +60,7 @@ import jakarta.ws.rs.core.HttpHeaders; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.Response.Status; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.jsonDT; import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; @@ -1048,4 +1049,26 @@ public Response getFileVersionsList(@Context ContainerRequestContext crc, @PathP return ex.getResponse(); } } + + @GET + @AuthRequired + @Path("{identifier}/permissions/history") + @Produces({ MediaType.APPLICATION_JSON, "text/csv" }) + public Response getRoleAssignmentHistory(@Context ContainerRequestContext crc, + @PathParam("identifier") String id, + @Context HttpHeaders headers) { + return response(req -> { + DataFile dataFile = findDataFileOrDie(id); + + // user is authenticated + AuthenticatedUser authenticatedUser = null; + try { + authenticatedUser = getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return error(Status.UNAUTHORIZED, "Authentication is required."); + } + + return getRoleAssignmentHistoryResponse(dataFile, authenticatedUser, headers); + }, getRequestUser(crc)); + } } From 0989e35999f93a260a92a2fe2777cb5f3c67806b Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 17 Jul 2025 13:46:53 -0400 Subject: [PATCH 044/634] download csv buttons --- .../webapp/permissions-manage-files.xhtml | 9 +++++++ src/main/webapp/permissions-manage.xhtml | 26 +++---------------- .../webapp/resources/js/rahistory_utils.js | 17 ++++++++++++ 3 files changed, 29 insertions(+), 23 deletions(-) create mode 100644 src/main/webapp/resources/js/rahistory_utils.js diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index 1a93527dfef..9aaf306532b 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -9,6 +9,7 @@ xmlns:jsf="http://xmlns.jcp.org/jsf" xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs"> + @@ -228,6 +229,14 @@
+ +
+ + #{bundle['dataverse.permissions.history.download']} + +
diff --git a/src/main/webapp/permissions-manage.xhtml b/src/main/webapp/permissions-manage.xhtml index ef9d9345b32..a860f19bc07 100644 --- a/src/main/webapp/permissions-manage.xhtml +++ b/src/main/webapp/permissions-manage.xhtml @@ -8,6 +8,7 @@ xmlns:o="http://omnifaces.org/ui" xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs"> + @@ -211,37 +212,16 @@ + onclick="downloadRAHistoryCSV(this.href, 'dataset_permissions_history.csv'); return false;"> #{bundle['dataverse.permissions.history.download']} + onclick="downloadRAHistoryCSV(this.href, 'dataverse_permissions_history.csv'); return false;"> #{bundle['dataverse.permissions.history.download']}
- - response.blob()) + .then(blob => { + const link = document.createElement('a'); + link.href = URL.createObjectURL(blob); + link.download = filename; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + }) + .catch(error => console.error('Error downloading CSV:', error)); +} \ No newline at end of file From db6e94b32a3f66d1436f24e665b36af2666b5573 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 17 Jul 2025 13:47:08 -0400 Subject: [PATCH 045/634] strings for csv headers and button --- src/main/java/propertyFiles/Bundle.properties | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 10cf777daf5..93bb79ff491 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1213,6 +1213,7 @@ dataverse.permissions.history.revokedBy=Revoked By dataverse.permissions.history.revokedAt=Revocation Date dataverse.permissions.history.definedOn=Defined On dataverse.permissions.history.files={0} Files +dataverse.permissions.history.download=Download (CSV) # permissions-manage-files.xhtml dataverse.permissionsFiles.title=Restricted File Permissions dataverse.permissionsFiles.usersOrGroups=Users/Groups @@ -3229,6 +3230,13 @@ updateDatasetFieldsCommand.api.processDatasetUpdate.parseError=Error parsing dat #AbstractApiBean.java abstractApiBean.error.datasetInternalVersionNumberIsOutdated=Dataset internal version number {0} is outdated +# Role Assignment History (used in the AbstractApiBean) +datasets.api.permissions.history.assignee=Assignee +datasets.api.permissions.history.role=Role +datasets.api.permissions.history.assignedBy=Assigned By +datasets.api.permissions.history.assignedAt=Assigned At +datasets.api.permissions.history.revokedBy=Revoked By +datasets.api.permissions.history.revokedAt=Revoked At #RoleAssigneeServiceBean.java roleAssigneeServiceBean.error.dataverseRequestCannotBeNull=DataverseRequest cannot be null. \ No newline at end of file From 7e468f44bc3e743a06fdf9d1c1b9cc1b4504d358 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 21 Jul 2025 17:24:21 -0400 Subject: [PATCH 046/634] try signed URLs for CSV buttons, fix files reporting/api --- .../dataverse/DataverseRoleServiceBean.java | 2 +- .../dataverse/ManageFilePermissionsPage.java | 44 +++++++++++++- .../iq/dataverse/ManagePermissionsPage.java | 60 +++++++++++++++++++ .../iq/dataverse/api/AbstractApiBean.java | 26 +++++--- .../harvard/iq/dataverse/api/Datasets.java | 29 +++++---- .../harvard/iq/dataverse/api/Dataverses.java | 2 +- .../edu/harvard/iq/dataverse/api/Files.java | 22 ------- src/main/webapp/permissions-manage.xhtml | 9 +-- .../webapp/resources/js/rahistory_utils.js | 24 +++++++- 9 files changed, 165 insertions(+), 53 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java index 68fee4aacad..257985dc65a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java @@ -393,7 +393,7 @@ public List getRoleAssignmentHistory(Long definition * @param datasetId The ID of the dataset * @return List of role assignment history entries */ - public List getChildRoleAssignmentHistory(Long datasetId) { + public List getFilesRoleAssignmentHistory(Long datasetId) { List audits = em.createNamedQuery("RoleAssignmentAudit.findByOwnerId", RoleAssignmentAudit.class) .setParameter("datasetId", datasetId) .getResultList(); diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java index 1cd59bbdf57..574ea0a9664 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java @@ -15,14 +15,20 @@ import edu.harvard.iq.dataverse.authorization.RoleAssigneeDisplayInfo; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroupServiceBean; +import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; import edu.harvard.iq.dataverse.engine.command.impl.AssignRoleCommand; import edu.harvard.iq.dataverse.engine.command.impl.RevokeRoleCommand; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.DateUtil; import edu.harvard.iq.dataverse.util.JsfHelper; +import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.UrlSignerUtil; + import static edu.harvard.iq.dataverse.util.JsfHelper.JH; import java.sql.Timestamp; import java.util.*; @@ -544,7 +550,7 @@ private boolean assignRole(RoleAssignee ra, DataFile file, DataverseRole r) { public List getRoleAssignmentHistory() { if (roleAssignmentHistory == null) { - roleAssignmentHistory = roleService.getChildRoleAssignmentHistory(dataset.getId()); + roleAssignmentHistory = roleService.getFilesRoleAssignmentHistory(dataset.getId()); } return roleAssignmentHistory; } @@ -578,7 +584,43 @@ public void setRenderFileMessages(boolean renderFileMessages) { this.renderFileMessages = renderFileMessages; } + public String getsignedUrlForRAHistoryCsv() { + String apiPath = "/api/v1/datasets/" + dataset.getId() + "/files/permissions/history"; + + try { + // Get the application URL from the system config + String baseUrl = SystemConfig.getDataverseSiteUrlStatic(); + if (baseUrl.endsWith("/")) { + baseUrl = baseUrl.substring(0, baseUrl.length() - 1); + } + + // Construct the full URL + String fullApiPath = baseUrl + apiPath; + + // Generate a signed URL with the user's API token + User user = session.getUser(); + String key = null; + if (user instanceof AuthenticatedUser) { + ApiToken apiToken = authenticationService.findApiTokenByUser((AuthenticatedUser) user); + if (apiToken != null && !apiToken.isExpired() && !apiToken.isDisabled()) { + key = apiToken.getTokenString(); + } + } + key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key; + if(key.length() >= 36) { + return UrlSignerUtil.signUrl(fullApiPath, 10, user.getIdentifier(), "GET", key); + } + } catch (Exception e) { + logger.log(Level.SEVERE, "Error generating signed URL for permissions history CSV: " + e.getMessage(), e); + return null; + } + return null; + } + public String getPermissionsHistoryFilename() { + // For datasets, replace colons in the PID with underscores + return dataset.getGlobalId().asString().replace(":", "_") + "_files_permissions_history.csv"; + } // inner class used fordisplay of role assignments diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index 4fbf520154b..2a08b64fed6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -11,17 +11,24 @@ import edu.harvard.iq.dataverse.authorization.groups.impl.builtin.AuthenticatedUsers; import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroup; import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroupServiceBean; +import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; import edu.harvard.iq.dataverse.engine.command.impl.AssignRoleCommand; import edu.harvard.iq.dataverse.engine.command.impl.CreateRoleCommand; import edu.harvard.iq.dataverse.engine.command.impl.RevokeRoleCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseDefaultContributorRoleCommand; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.JsfHelper; import static edu.harvard.iq.dataverse.util.JsfHelper.JH; import edu.harvard.iq.dataverse.util.StringUtil; +import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.URLTokenUtil; +import edu.harvard.iq.dataverse.util.UrlSignerUtil; + import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; @@ -688,6 +695,59 @@ public Boolean getRenderRoleMessages() { public void setRenderRoleMessages(Boolean renderRoleMessages) { this.renderRoleMessages = renderRoleMessages; } + + public String getsignedUrlForRAHistoryCsv() { + String apiPath; + + if (dvObject instanceof Dataverse dv) { + // For Dataverses, use the dataverses API endpoint with the alias + apiPath = "/api/v1/dataverses/" + dv.getAlias() + "/permissions/history"; + } else if (dvObject instanceof Dataset) { + // For Datasets, use the datasets API endpoint with the ID + apiPath = "/api/v1/datasets/" + dvObject.getId() + "/permissions/history"; + } else { + // For other types (like DataFile), return null or a default path + return null; + } + + try { + // Get the application URL from the system config + String baseUrl = SystemConfig.getDataverseSiteUrlStatic(); + if (baseUrl.endsWith("/")) { + baseUrl = baseUrl.substring(0, baseUrl.length() - 1); + } + + // Construct the full URL + String fullApiPath = baseUrl + apiPath; + + // Generate a signed URL with the user's API token + User user = session.getUser(); + String key = null; + if (user instanceof AuthenticatedUser) { + ApiToken apiToken = authenticationService.findApiTokenByUser((AuthenticatedUser) user); + if (apiToken != null && !apiToken.isExpired() && !apiToken.isDisabled()) { + key = apiToken.getTokenString(); + } + } + key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key; + if(key.length() >= 36) { + return UrlSignerUtil.signUrl(fullApiPath, 10, user.getIdentifier(), "GET", key); + } + } catch (Exception e) { + logger.log(Level.SEVERE, "Error generating signed URL for permissions history CSV: " + e.getMessage(), e); + return null; + } + return null; + } + + public String getPermissionsHistoryFilename() { + if (dvObject instanceof Dataverse dv) { + return dv.getAlias() + "_permissions_history.csv"; + } else { + // For datasets, replace colons in the PID with underscores + return dvObject.getGlobalId().asString().replace(":", "_") + "_permissions_history.csv"; + } + } // inner class used for display of role assignments public static class RoleAssignmentRow { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index a6562b7c1d9..afbd82dedc2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -654,14 +654,22 @@ protected DatasetFieldType findDatasetFieldType(String idtf) throws NumberFormat * @param headers HTTP headers from the request (for content negotiation) * @return Response containing history in JSON or CSV format */ - protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, AuthenticatedUser authenticatedUser, HttpHeaders headers) { + protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, AuthenticatedUser authenticatedUser, boolean forFiles, HttpHeaders headers) { // Check if the user has permission to manage permissions for this object if (!permissionSvc.userOn(authenticatedUser, dvObject).has(Permission.ManageDatasetPermissions)) { return error(Status.FORBIDDEN, "You do not have permission to view the role assignment history for this " + dvObject.getClass().getSimpleName().toLowerCase()); } // Get the role assignment history - List history = rolesSvc.getRoleAssignmentHistory(dvObject.getId()); + List history = null; + String definitionPoint; + if (forFiles == false) { + history = rolesSvc.getRoleAssignmentHistory(dvObject.getId()); + definitionPoint = BundleUtil.getStringFromBundle("datasets.api.permissions.history.definitionPoint"); + } else { + history = rolesSvc.getFilesRoleAssignmentHistory(dvObject.getId()); + definitionPoint = BundleUtil.getStringFromBundle("datasets.api.permissions.history.definitionPoints"); + } List acceptedTypes = headers.getAcceptableMediaTypes(); boolean wantCSV = acceptedTypes.stream() @@ -678,7 +686,9 @@ protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, Authentic // Generate CSV response StringBuilder csvBuilder = new StringBuilder(); // Add CSV header with internationalized column names - csvBuilder.append(assigneeHeader).append(",") + csvBuilder + .append(definitionPoint).append(",") + .append(assigneeHeader).append(",") .append(roleHeader).append(",") .append(assignedByHeader).append(",") .append(assignedAtHeader).append(",") @@ -687,10 +697,11 @@ protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, Authentic // Add data rows for (DataverseRoleServiceBean.RoleAssignmentHistoryEntry entry : history) { - csvBuilder.append(entry.getAssigneeIdentifier()).append(","); - csvBuilder.append(entry.getRoleName()).append(","); - csvBuilder.append(entry.getAssignedBy()).append(","); - csvBuilder.append(entry.getAssignedAt().toString()).append(","); + csvBuilder.append(entry.getDefinitionPointIdsAsString()).append(",") + .append(entry.getAssigneeIdentifier()).append(",") + .append(entry.getRoleName()).append(",") + .append(entry.getAssignedBy()).append(",") + .append(entry.getAssignedAt().toString()).append(","); // Handle nullable fields csvBuilder.append(entry.getRevokedBy() != null ? entry.getRevokedBy() : "").append(","); @@ -710,6 +721,7 @@ protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, Authentic JsonArrayBuilder jsonArray = Json.createArrayBuilder(); for (DataverseRoleServiceBean.RoleAssignmentHistoryEntry entry : history) { JsonObjectBuilder job = Json.createObjectBuilder() + .add(definitionPoint, entry.getDefinitionPointIdsAsString()) .add("assigneeIdentifier", entry.getAssigneeIdentifier()) .add("roleName", entry.getRoleName()) .add("assignedBy", entry.getAssignedBy()) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index e657ac02175..e2b8c7f5c80 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -5,6 +5,7 @@ import edu.harvard.iq.dataverse.DatasetVersion.VersionState; import edu.harvard.iq.dataverse.DataverseRoleServiceBean.RoleAssignmentHistoryEntry; import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord; +import edu.harvard.iq.dataverse.api.AbstractApiBean.WrappedResponse; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.api.dto.RoleAssignmentDTO; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; @@ -5985,23 +5986,31 @@ public Response deleteVersionNote(@Context ContainerRequestContext crc, @PathPar @GET @AuthRequired @Path("{identifier}/permissions/history") + public Response getRoleAssignmentHistory(@Context ContainerRequestContext crc, @PathParam("identifier") String id, @Context HttpHeaders headers) { + return response(req -> { + Dataset dataset = findDatasetOrDie(id); + + // user is authenticated + AuthenticatedUser authenticatedUser = getRequestAuthenticatedUserOrDie(crc); + + return getRoleAssignmentHistoryResponse(dataset, authenticatedUser, false, headers); + }, getRequestUser(crc)); + } + + @GET + @AuthRequired + @Path("{identifier}/files/permissions/history") @Produces({ MediaType.APPLICATION_JSON, "text/csv" }) - public Response getRoleAssignmentHistory(@Context ContainerRequestContext crc, + public Response getFilesRoleAssignmentHistory(@Context ContainerRequestContext crc, @PathParam("identifier") String id, @Context HttpHeaders headers) { return response(req -> { Dataset dataset = findDatasetOrDie(id); - + // user is authenticated - AuthenticatedUser authenticatedUser = null; - try { - authenticatedUser = getRequestAuthenticatedUserOrDie(crc); - } catch (WrappedResponse ex) { - return error(Status.UNAUTHORIZED, "Authentication is required."); - } + AuthenticatedUser authenticatedUser = getRequestAuthenticatedUserOrDie(crc); - return getRoleAssignmentHistoryResponse(dataset, authenticatedUser, headers); + return getRoleAssignmentHistoryResponse(dataset, authenticatedUser, true, headers); }, getRequestUser(crc)); } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index cc850dee1f4..0747dd1b994 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1951,7 +1951,7 @@ public Response getRoleAssignmentHistory(@Context ContainerRequestContext crc, return error(Status.UNAUTHORIZED, "Authentication is required."); } - return getRoleAssignmentHistoryResponse(dataverse, authenticatedUser, headers); + return getRoleAssignmentHistoryResponse(dataverse, authenticatedUser, false, headers); }, getRequestUser(crc)); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 59470530047..779f043fb7c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -1049,26 +1049,4 @@ public Response getFileVersionsList(@Context ContainerRequestContext crc, @PathP return ex.getResponse(); } } - - @GET - @AuthRequired - @Path("{identifier}/permissions/history") - @Produces({ MediaType.APPLICATION_JSON, "text/csv" }) - public Response getRoleAssignmentHistory(@Context ContainerRequestContext crc, - @PathParam("identifier") String id, - @Context HttpHeaders headers) { - return response(req -> { - DataFile dataFile = findDataFileOrDie(id); - - // user is authenticated - AuthenticatedUser authenticatedUser = null; - try { - authenticatedUser = getRequestAuthenticatedUserOrDie(crc); - } catch (WrappedResponse ex) { - return error(Status.UNAUTHORIZED, "Authentication is required."); - } - - return getRoleAssignmentHistoryResponse(dataFile, authenticatedUser, headers); - }, getRequestUser(crc)); - } } diff --git a/src/main/webapp/permissions-manage.xhtml b/src/main/webapp/permissions-manage.xhtml index a860f19bc07..ee8bd263f2e 100644 --- a/src/main/webapp/permissions-manage.xhtml +++ b/src/main/webapp/permissions-manage.xhtml @@ -209,18 +209,11 @@
- #{bundle['dataverse.permissions.history.download']} - - #{bundle['dataverse.permissions.history.download']} -
response.blob()) + .then(response => { + // Check if response is ok (status in the range 200-299) + if (!response.ok) { + throw new Error('Network response was not ok: ' + response.status); + } + return response.blob(); + }) .then(blob => { const link = document.createElement('a'); link.href = URL.createObjectURL(blob); - link.download = filename; + link.download = filename || 'permissions_history.csv'; // Provide default filename if none provided document.body.appendChild(link); link.click(); document.body.removeChild(link); }) - .catch(error => console.error('Error downloading CSV:', error)); + .catch(error => { + console.error('Error downloading CSV:', error); + // Display a user-friendly error message + alert('Failed to download permissions history. Please try again later.'); + }); } \ No newline at end of file From 13ac931ef0c6ea99816a47acb1880b8757768507 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 23 Jul 2025 18:31:41 -0400 Subject: [PATCH 047/634] use getUserIdentifier() --- .../edu/harvard/iq/dataverse/ManagePermissionsPage.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index 2a08b64fed6..2a6d2b024a8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -723,15 +723,18 @@ public String getsignedUrlForRAHistoryCsv() { // Generate a signed URL with the user's API token User user = session.getUser(); String key = null; - if (user instanceof AuthenticatedUser) { - ApiToken apiToken = authenticationService.findApiTokenByUser((AuthenticatedUser) user); + String userId=null; + if (user instanceof AuthenticatedUser aUser) { + userId = aUser.getUserIdentifier(); + ApiToken apiToken = authenticationService.findApiTokenByUser(aUser); + if (apiToken != null && !apiToken.isExpired() && !apiToken.isDisabled()) { key = apiToken.getTokenString(); } } key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key; if(key.length() >= 36) { - return UrlSignerUtil.signUrl(fullApiPath, 10, user.getIdentifier(), "GET", key); + return UrlSignerUtil.signUrl(fullApiPath, 10, userId, "GET", key); } } catch (Exception e) { logger.log(Level.SEVERE, "Error generating signed URL for permissions history CSV: " + e.getMessage(), e); From 7ed23bf632ab903234b15e055e94c74b5d552089 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 24 Jul 2025 10:32:24 -0400 Subject: [PATCH 048/634] add missing strings --- src/main/java/propertyFiles/Bundle.properties | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 8d65b3b0689..6caabb3c818 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -3232,6 +3232,8 @@ updateDatasetFieldsCommand.api.processDatasetUpdate.parseError=Error parsing dat #AbstractApiBean.java abstractApiBean.error.datasetInternalVersionNumberIsOutdated=Dataset internal version number {0} is outdated # Role Assignment History (used in the AbstractApiBean) +datasets.api.permissions.history.definitionPoint=Definition Point +datasets.api.permissions.history.definitionPoints=Definition Points datasets.api.permissions.history.assignee=Assignee datasets.api.permissions.history.role=Role datasets.api.permissions.history.assignedBy=Assigned By From 94235c7b826cdc88be6f4f3f6e1ebdbfd87b6f9f Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 24 Jul 2025 10:32:50 -0400 Subject: [PATCH 049/634] update files page, use new filenames --- src/main/webapp/permissions-manage-files.xhtml | 6 +++--- src/main/webapp/permissions-manage.xhtml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index 9aaf306532b..1572d5c703a 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -229,11 +229,11 @@
- +
- + onclick="downloadRAHistoryCSV(this.href, '#{manageFilePermissionsPage.permissionsHistoryFilename}'); return false;"> #{bundle['dataverse.permissions.history.download']}
diff --git a/src/main/webapp/permissions-manage.xhtml b/src/main/webapp/permissions-manage.xhtml index ee8bd263f2e..a47820e0b90 100644 --- a/src/main/webapp/permissions-manage.xhtml +++ b/src/main/webapp/permissions-manage.xhtml @@ -211,7 +211,7 @@
+ onclick="downloadRAHistoryCSV(this.href, '#{managePermissionsPage.permissionsHistoryFilename}'); return false;"> #{bundle['dataverse.permissions.history.download']}
From aad80c4270722dff9529e60501556506386d715a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 24 Jul 2025 10:46:44 -0400 Subject: [PATCH 050/634] fix file url generation - authUser --- .../harvard/iq/dataverse/ManageFilePermissionsPage.java | 8 +++++--- .../edu/harvard/iq/dataverse/ManagePermissionsPage.java | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java index 574ea0a9664..a571f798060 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java @@ -600,15 +600,17 @@ public String getsignedUrlForRAHistoryCsv() { // Generate a signed URL with the user's API token User user = session.getUser(); String key = null; - if (user instanceof AuthenticatedUser) { - ApiToken apiToken = authenticationService.findApiTokenByUser((AuthenticatedUser) user); + String userId=null; + if (user instanceof AuthenticatedUser authUser) { + userId = authUser.getUserIdentifier(); + ApiToken apiToken = authenticationService.findApiTokenByUser(authUser); if (apiToken != null && !apiToken.isExpired() && !apiToken.isDisabled()) { key = apiToken.getTokenString(); } } key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key; if(key.length() >= 36) { - return UrlSignerUtil.signUrl(fullApiPath, 10, user.getIdentifier(), "GET", key); + return UrlSignerUtil.signUrl(fullApiPath, 10, userId, "GET", key); } } catch (Exception e) { logger.log(Level.SEVERE, "Error generating signed URL for permissions history CSV: " + e.getMessage(), e); diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index 2a6d2b024a8..ed1665f4208 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -724,9 +724,9 @@ public String getsignedUrlForRAHistoryCsv() { User user = session.getUser(); String key = null; String userId=null; - if (user instanceof AuthenticatedUser aUser) { - userId = aUser.getUserIdentifier(); - ApiToken apiToken = authenticationService.findApiTokenByUser(aUser); + if (user instanceof AuthenticatedUser authUser) { + userId = authUser.getUserIdentifier(); + ApiToken apiToken = authenticationService.findApiTokenByUser(authUser); if (apiToken != null && !apiToken.isExpired() && !apiToken.isDisabled()) { key = apiToken.getTokenString(); From 233f3e476e401cb779d798a07c3a66eac98702e7 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 24 Jul 2025 13:43:59 -0400 Subject: [PATCH 051/634] add produces annotation --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index e2b8c7f5c80..9054dda4d58 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -5986,6 +5986,7 @@ public Response deleteVersionNote(@Context ContainerRequestContext crc, @PathPar @GET @AuthRequired @Path("{identifier}/permissions/history") + @Produces({ MediaType.APPLICATION_JSON, "text/csv" }) public Response getRoleAssignmentHistory(@Context ContainerRequestContext crc, @PathParam("identifier") String id, @Context HttpHeaders headers) { return response(req -> { Dataset dataset = findDatasetOrDie(id); From c0897a9d1b2a4a5b90ea601bfba2375574d0f970 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 24 Jul 2025 14:36:10 -0400 Subject: [PATCH 052/634] move script to end of body --- src/main/webapp/permissions-manage-files.xhtml | 5 ++--- src/main/webapp/permissions-manage.xhtml | 4 +--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index 1572d5c703a..80e0a4eb1d3 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -8,9 +8,7 @@ xmlns:o="http://omnifaces.org/ui" xmlns:jsf="http://xmlns.jcp.org/jsf" xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs"> - - - + @@ -434,6 +432,7 @@
+ diff --git a/src/main/webapp/permissions-manage.xhtml b/src/main/webapp/permissions-manage.xhtml index a47820e0b90..354b963be6c 100644 --- a/src/main/webapp/permissions-manage.xhtml +++ b/src/main/webapp/permissions-manage.xhtml @@ -7,9 +7,6 @@ xmlns:c="http://java.sun.com/jsp/jstl/core" xmlns:o="http://omnifaces.org/ui" xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs"> - - - @@ -283,6 +280,7 @@
+ From 59c0680a8848d892494d98107ba3aec8e13d685d Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 24 Jul 2025 14:36:26 -0400 Subject: [PATCH 053/634] handle commas w/multiple def pts --- .../java/edu/harvard/iq/dataverse/api/AbstractApiBean.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index afbd82dedc2..8cdca1956d2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -697,6 +697,11 @@ protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, Authentic // Add data rows for (DataverseRoleServiceBean.RoleAssignmentHistoryEntry entry : history) { + String definitionPointIds = entry.getDefinitionPointIdsAsString(); + // Handle multiple comma-separated values in definitionPointIds column + if(definitionPointIds.contains(",")) { + definitionPointIds = "\"" + definitionPointIds + "\""; + } csvBuilder.append(entry.getDefinitionPointIdsAsString()).append(",") .append(entry.getAssigneeIdentifier()).append(",") .append(entry.getRoleName()).append(",") From c16730747d3f7d554bc1bcb63032e1d241741ec5 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 24 Jul 2025 15:00:23 -0400 Subject: [PATCH 054/634] reuse strings --- .../iq/dataverse/api/AbstractApiBean.java | 21 +++++++++---------- src/main/java/propertyFiles/Bundle.properties | 9 -------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index e45598ebc6e..9f8a9e77c94 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -676,13 +676,10 @@ protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, Authentic // Get the role assignment history List history = null; - String definitionPoint; if (forFiles == false) { history = rolesSvc.getRoleAssignmentHistory(dvObject.getId()); - definitionPoint = BundleUtil.getStringFromBundle("datasets.api.permissions.history.definitionPoint"); } else { history = rolesSvc.getFilesRoleAssignmentHistory(dvObject.getId()); - definitionPoint = BundleUtil.getStringFromBundle("datasets.api.permissions.history.definitionPoints"); } List acceptedTypes = headers.getAcceptableMediaTypes(); @@ -690,18 +687,20 @@ protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, Authentic .anyMatch(mt -> mt.toString().equals("text/csv")); if (wantCSV) { - String assigneeHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.assignee"); - String roleHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.role"); - String assignedByHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.assignedBy"); - String assignedAtHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.assignedAt"); - String revokedByHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.revokedBy"); - String revokedAtHeader = BundleUtil.getStringFromBundle("datasets.api.permissions.history.revokedAt"); + //Reusing strings from history panel + String definedOn = BundleUtil.getStringFromBundle("dataverse.permissions.history.definedOn"); + String assigneeHeader = BundleUtil.getStringFromBundle("dataverse.permissions.history.assignee"); + String roleHeader = BundleUtil.getStringFromBundle("dataverse.permissions.history.role"); + String assignedByHeader = BundleUtil.getStringFromBundle("dataverse.permissions.history.assignedBy"); + String assignedAtHeader = BundleUtil.getStringFromBundle("dataverse.permissions.history.assignedAt"); + String revokedByHeader = BundleUtil.getStringFromBundle("dataverse.permissions.history.revokedBy"); + String revokedAtHeader = BundleUtil.getStringFromBundle("dataverse.permissions.history.revokedAt"); // Generate CSV response StringBuilder csvBuilder = new StringBuilder(); // Add CSV header with internationalized column names csvBuilder - .append(definitionPoint).append(",") + .append(definedOn).append(",") .append(assigneeHeader).append(",") .append(roleHeader).append(",") .append(assignedByHeader).append(",") @@ -740,7 +739,7 @@ protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, Authentic JsonArrayBuilder jsonArray = Json.createArrayBuilder(); for (DataverseRoleServiceBean.RoleAssignmentHistoryEntry entry : history) { JsonObjectBuilder job = Json.createObjectBuilder() - .add(definitionPoint, entry.getDefinitionPointIdsAsString()) + .add("definedOn", entry.getDefinitionPointIdsAsString()) .add("assigneeIdentifier", entry.getAssigneeIdentifier()) .add("roleName", entry.getRoleName()) .add("assignedBy", entry.getAssignedBy()) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 381ba878844..ede0bbb8f8a 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -3232,15 +3232,6 @@ datasetFieldValidator.error.emptyRequiredSingleValueForField=Empty required valu updateDatasetFieldsCommand.api.processDatasetUpdate.parseError=Error parsing dataset update: {0} #AbstractApiBean.java -# Role Assignment History (used in the AbstractApiBean) -datasets.api.permissions.history.definitionPoint=Definition Point -datasets.api.permissions.history.definitionPoints=Definition Points -datasets.api.permissions.history.assignee=Assignee -datasets.api.permissions.history.role=Role -datasets.api.permissions.history.assignedBy=Assigned By -datasets.api.permissions.history.assignedAt=Assigned At -datasets.api.permissions.history.revokedBy=Revoked By -datasets.api.permissions.history.revokedAt=Revoked At abstractApiBean.error.internalVersionTimestampIsOutdated=Internal version timestamp {0} is outdated #RoleAssigneeServiceBean.java From f10dcbbc4e0385dc93cb5d4283867f5b16ce68c8 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 24 Jul 2025 17:10:36 -0400 Subject: [PATCH 055/634] handle missing assignment info --- .../iq/dataverse/api/AbstractApiBean.java | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 9f8a9e77c94..a9bb064f487 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -718,13 +718,11 @@ protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, Authentic csvBuilder.append(entry.getDefinitionPointIdsAsString()).append(",") .append(entry.getAssigneeIdentifier()).append(",") .append(entry.getRoleName()).append(",") - .append(entry.getAssignedBy()).append(",") - .append(entry.getAssignedAt().toString()).append(","); - - // Handle nullable fields - csvBuilder.append(entry.getRevokedBy() != null ? entry.getRevokedBy() : "").append(","); - csvBuilder.append(entry.getRevokedAt() != null ? entry.getRevokedAt().toString() : ""); - csvBuilder.append("\n"); + .append(entry.getAssignedBy() != null ? entry.getAssignedBy() : "").append(",") + .append(entry.getAssignedAt() != null ? entry.getAssignedAt().toString() : "").append(",") + .append(entry.getRevokedBy() != null ? entry.getRevokedBy() : "").append(",") + .append(entry.getRevokedAt() != null ? entry.getRevokedAt().toString() : "") + .append("\n"); } String objectType = dvObject.getClass().getSimpleName().toLowerCase(); @@ -741,9 +739,19 @@ protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, Authentic JsonObjectBuilder job = Json.createObjectBuilder() .add("definedOn", entry.getDefinitionPointIdsAsString()) .add("assigneeIdentifier", entry.getAssigneeIdentifier()) - .add("roleName", entry.getRoleName()) - .add("assignedBy", entry.getAssignedBy()) - .add("assignedAt", entry.getAssignedAt().toString()); + .add("roleName", entry.getRoleName()); + + // Add assignment info if available + if (entry.getAssignedBy()!= null) { + job.add("assignedBy", entry.getAssignedBy()); + } else { + job.add("assignedBy", JsonValue.NULL); + } + if (entry.getAssignedAt()!= null) { + job.add("assignedAt", entry.getAssignedAt().toString()); + } else { + job.add("assignedAt", JsonValue.NULL); + } // Add revocation info if available if (entry.getRevokedBy() != null) { @@ -751,7 +759,6 @@ protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, Authentic } else { job.add("revokedBy", JsonValue.NULL); } - if (entry.getRevokedAt() != null) { job.add("revokedAt", entry.getRevokedAt().toString()); } else { From 080372639a499855d20d5fb906147f317834fd47 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 24 Jul 2025 15:03:23 -0400 Subject: [PATCH 056/634] fix comma handling --- src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index a9bb064f487..4c3b99423d6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -715,7 +715,7 @@ protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, Authentic if(definitionPointIds.contains(",")) { definitionPointIds = "\"" + definitionPointIds + "\""; } - csvBuilder.append(entry.getDefinitionPointIdsAsString()).append(",") + csvBuilder.append(definitionPointIds).append(",") .append(entry.getAssigneeIdentifier()).append(",") .append(entry.getRoleName()).append(",") .append(entry.getAssignedBy() != null ? entry.getAssignedBy() : "").append(",") From 715badb279b287f327bd141fec30fa70d5519dad Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 24 Jul 2025 17:39:42 -0400 Subject: [PATCH 057/634] change api paths to mirror /assignment endpoints --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 91239af8816..03c9b53ce4a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -6023,7 +6023,7 @@ public Response deleteVersionNote(@Context ContainerRequestContext crc, @PathPar @GET @AuthRequired - @Path("{identifier}/permissions/history") + @Path("{identifier}/assignments/history") @Produces({ MediaType.APPLICATION_JSON, "text/csv" }) public Response getRoleAssignmentHistory(@Context ContainerRequestContext crc, @PathParam("identifier") String id, @Context HttpHeaders headers) { return response(req -> { @@ -6038,7 +6038,7 @@ public Response getRoleAssignmentHistory(@Context ContainerRequestContext crc, @ @GET @AuthRequired - @Path("{identifier}/files/permissions/history") + @Path("{identifier}/files/assignments/history") @Produces({ MediaType.APPLICATION_JSON, "text/csv" }) public Response getFilesRoleAssignmentHistory(@Context ContainerRequestContext crc, @PathParam("identifier") String id, From baf5e180a236977f4b84509e5bb71f3079046d9a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 25 Jul 2025 11:52:45 -0400 Subject: [PATCH 058/634] change flag name, class names to remove audit --- .../dataverse/DataverseRoleServiceBean.java | 28 +++++++++---------- .../dataverse/ManageFilePermissionsPage.java | 6 ++-- .../iq/dataverse/ManagePermissionsPage.java | 4 +-- ...tAudit.java => RoleAssignmentHistory.java} | 14 +++++----- .../iq/dataverse/api/AbstractApiBean.java | 6 ++-- .../harvard/iq/dataverse/api/Datasets.java | 2 +- .../command/impl/AssignRoleCommand.java | 2 +- .../iq/dataverse/settings/FeatureFlags.java | 4 +-- 8 files changed, 33 insertions(+), 33 deletions(-) rename src/main/java/edu/harvard/iq/dataverse/{RoleAssignmentAudit.java => RoleAssignmentHistory.java} (89%) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java index 257985dc65a..78a6628ec3a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java @@ -94,7 +94,7 @@ public RoleAssignment save(RoleAssignment assignment, boolean createIndex, Datav // Check if ROLE_ASSIGNMENT_AUDITING feature flag is enabled if (FeatureFlags.ROLE_ASSIGNMENT_AUDITING.enabled()) { - RoleAssignmentAudit audit = new RoleAssignmentAudit(assignment, req, RoleAssignmentAudit.ActionType.ASSIGN); + RoleAssignmentHistory audit = new RoleAssignmentHistory(assignment, req, RoleAssignmentHistory.ActionType.ASSIGN); saveAudit(audit); } @@ -103,12 +103,12 @@ public RoleAssignment save(RoleAssignment assignment, boolean createIndex, Datav /** - * Saves a RoleAssignmentAudit entry to the database. + * Saves a RoleAssignmentHistory entry to the database. * - * @param audit The RoleAssignmentAudit object to be saved - * @return The persisted RoleAssignmentAudit object + * @param audit The RoleAssignmentHistory object to be saved + * @return The persisted RoleAssignmentHistory object */ - private RoleAssignmentAudit saveAudit(RoleAssignmentAudit audit) { + private RoleAssignmentHistory saveAudit(RoleAssignmentHistory audit) { if (audit.getAuditId() == null) { em.persist(audit); em.flush(); // Ensure the entity is persisted immediately @@ -184,7 +184,7 @@ public void revoke(RoleAssignment ra, DataverseRequest req) { // Create audit entry if feature flag is set if (FeatureFlags.ROLE_ASSIGNMENT_AUDITING.enabled()) { - RoleAssignmentAudit audit = new RoleAssignmentAudit(ra, req, RoleAssignmentAudit.ActionType.REVOKE); + RoleAssignmentHistory audit = new RoleAssignmentHistory(ra, req, RoleAssignmentHistory.ActionType.REVOKE); saveAudit(audit); } @@ -209,7 +209,7 @@ public void revokeAll(RoleAssignee assignee, DataverseRequest req) { // Create audit entry if feature flag is set if (FeatureFlags.ROLE_ASSIGNMENT_AUDITING.enabled()) { - RoleAssignmentAudit audit = new RoleAssignmentAudit(ra, req, RoleAssignmentAudit.ActionType.REVOKE); + RoleAssignmentHistory audit = new RoleAssignmentHistory(ra, req, RoleAssignmentHistory.ActionType.REVOKE); saveAudit(audit); } @@ -380,7 +380,7 @@ For a given permission and dataverse Id get all of the roles (built-in or owned * @return List of role assignment history entries */ public List getRoleAssignmentHistory(Long definitionPointId) { - List audits = em.createNamedQuery("RoleAssignmentAudit.findByDefinitionPointId", RoleAssignmentAudit.class) + List audits = em.createNamedQuery("RoleAssignmentHistory.findByDefinitionPointId", RoleAssignmentHistory.class) .setParameter("definitionPointId", definitionPointId) .getResultList(); @@ -394,7 +394,7 @@ public List getRoleAssignmentHistory(Long definition * @return List of role assignment history entries */ public List getFilesRoleAssignmentHistory(Long datasetId) { - List audits = em.createNamedQuery("RoleAssignmentAudit.findByOwnerId", RoleAssignmentAudit.class) + List audits = em.createNamedQuery("RoleAssignmentHistory.findByOwnerId", RoleAssignmentHistory.class) .setParameter("datasetId", datasetId) .getResultList(); @@ -402,18 +402,18 @@ public List getFilesRoleAssignmentHistory(Long datas } /** - * Common method to process role assignment audits and create history entries + * Common method to process role assignment history rows and create consolidated history entries * * @param audits List of role assignment audit records * @param combineEntries Whether to combine entries for different files * @return List of role assignment history entries */ - private List processRoleAssignmentAudits(List audits, boolean combineEntries) { + private List processRoleAssignmentAudits(List audits, boolean combineEntries) { List roleAssignmentHistory = new ArrayList<>(); Map historyMap = new HashMap<>(); // First pass: Create entries from audit records - for (RoleAssignmentAudit audit : audits) { + for (RoleAssignmentHistory audit : audits) { Long roleAssignmentId = audit.getRoleAssignmentId(); RoleAssignmentHistoryEntry entry = historyMap.get(roleAssignmentId); @@ -422,10 +422,10 @@ private List processRoleAssignmentAudits(List roleAssignmentHistory; + private List roleAssignmentHistory; - public List getRoleAssignmentHistory() { + public List getRoleAssignmentHistory() { if (roleAssignmentHistory == null) { roleAssignmentHistory = roleService.getFilesRoleAssignmentHistory(dataset.getId()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index ed1665f4208..1f980c877ba 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -241,9 +241,9 @@ public void editRole(String roleId) { } /** Role Assignment History */ - private List roleAssignmentHistory; + private List roleAssignmentHistory; - public List getRoleAssignmentHistory() { + public List getRoleAssignmentHistory() { if (roleAssignmentHistory == null) { roleAssignmentHistory = roleService.getRoleAssignmentHistory(dvObject.getId()); diff --git a/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentHistory.java similarity index 89% rename from src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java rename to src/main/java/edu/harvard/iq/dataverse/RoleAssignmentHistory.java index 52292aeb065..ebc6d95288a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentAudit.java +++ b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentHistory.java @@ -30,14 +30,14 @@ @Index(name = "idx_raa_definition_point_id", columnList = "definition_point_id") }) @NamedQueries({ - @NamedQuery(name = "RoleAssignmentAudit.findByDefinitionPointId", - query = "SELECT ra FROM RoleAssignmentAudit ra WHERE ra.definitionPointId = :definitionPointId ORDER BY ra.roleAssignmentId, ra.actionTimestamp DESC"), - @NamedQuery(name = "RoleAssignmentAudit.findByOwnerId", - query = "SELECT ra FROM RoleAssignmentAudit ra JOIN DvObject d ON ra.definitionPointId = d.id " + + @NamedQuery(name = "RoleAssignmentHistory.findByDefinitionPointId", + query = "SELECT ra FROM RoleAssignmentHistory ra WHERE ra.definitionPointId = :definitionPointId ORDER BY ra.roleAssignmentId, ra.actionTimestamp DESC"), + @NamedQuery(name = "RoleAssignmentHistory.findByOwnerId", + query = "SELECT ra FROM RoleAssignmentHistory ra JOIN DvObject d ON ra.definitionPointId = d.id " + "WHERE d.owner.id = :datasetId " + "ORDER BY ra.roleAssignmentId, ra.actionTimestamp DESC") }) -public class RoleAssignmentAudit implements Serializable { +public class RoleAssignmentHistory implements Serializable { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @@ -78,10 +78,10 @@ public enum ActionType { } // Constructors - public RoleAssignmentAudit() { + public RoleAssignmentHistory() { } - public RoleAssignmentAudit(RoleAssignment roleAssignment, DataverseRequest request, ActionType actionType) { + public RoleAssignmentHistory(RoleAssignment roleAssignment, DataverseRequest request, ActionType actionType) { this.roleAssignmentId = roleAssignment.getId(); this.actionType = actionType; this.actionTimestamp = new Date(); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 4c3b99423d6..fb5868d76b0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -675,7 +675,7 @@ protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, Authentic } // Get the role assignment history - List history = null; + List history = null; if (forFiles == false) { history = rolesSvc.getRoleAssignmentHistory(dvObject.getId()); } else { @@ -709,7 +709,7 @@ protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, Authentic .append(revokedAtHeader).append("\n"); // Add data rows - for (DataverseRoleServiceBean.RoleAssignmentHistoryEntry entry : history) { + for (DataverseRoleServiceBean.RoleAssignmentHistoryConsolidatedEntry entry : history) { String definitionPointIds = entry.getDefinitionPointIdsAsString(); // Handle multiple comma-separated values in definitionPointIds column if(definitionPointIds.contains(",")) { @@ -735,7 +735,7 @@ protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, Authentic // Or Json by default JsonArrayBuilder jsonArray = Json.createArrayBuilder(); - for (DataverseRoleServiceBean.RoleAssignmentHistoryEntry entry : history) { + for (DataverseRoleServiceBean.RoleAssignmentHistoryConsolidatedEntry entry : history) { JsonObjectBuilder job = Json.createObjectBuilder() .add("definedOn", entry.getDefinitionPointIdsAsString()) .add("assigneeIdentifier", entry.getAssigneeIdentifier()) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 03c9b53ce4a..6dd0baaa2c7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3,7 +3,7 @@ import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.DatasetLock.Reason; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; -import edu.harvard.iq.dataverse.DataverseRoleServiceBean.RoleAssignmentHistoryEntry; +import edu.harvard.iq.dataverse.DataverseRoleServiceBean.RoleAssignmentHistoryConsolidatedEntry; import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord; import edu.harvard.iq.dataverse.api.AbstractApiBean.WrappedResponse; import edu.harvard.iq.dataverse.api.auth.AuthRequired; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java index 7260b433ddb..db3c41f0f03 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java @@ -19,7 +19,7 @@ import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.util.BundleUtil; -import edu.harvard.iq.dataverse.RoleAssignmentAudit; +import edu.harvard.iq.dataverse.RoleAssignmentHistory; import edu.harvard.iq.dataverse.settings.FeatureFlags; import java.util.Collections; import java.util.Date; diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java index 539e898255c..7d3fec6822f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java @@ -168,10 +168,10 @@ public enum FeatureFlags { ADD_LOCAL_CONTEXTS_PERMISSION_CHECK("add-local-contexts-permission-check"), /** - * This flag turns on auditing of role assignments - keeping a record of when roles were granted + * This flag turns on history tracking of role assignments - keeping a record of when roles were granted * or revoked, at what times, and by whom. */ - ROLE_ASSIGNMENT_AUDITING("role-assignment-auditing"), + ROLE_ASSIGNMENT_HISTORY("role-assignment-history"), ; final String flag; From 696f009201567c1950008040ee6810d8fd867f1c Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 25 Jul 2025 11:53:05 -0400 Subject: [PATCH 059/634] more audit changes --- .../dataverse/DataverseRoleServiceBean.java | 104 +++++++++--------- .../iq/dataverse/RoleAssignmentHistory.java | 2 +- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java index 78a6628ec3a..3b058c00c82 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java @@ -1,6 +1,6 @@ package edu.harvard.iq.dataverse; -import edu.harvard.iq.dataverse.DataverseRoleServiceBean.RoleAssignmentHistoryEntry; +import edu.harvard.iq.dataverse.DataverseRoleServiceBean.RoleAssignmentHistoryConsolidatedEntry; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.RoleAssignee; @@ -92,10 +92,10 @@ public RoleAssignment save(RoleAssignment assignment, boolean createIndex, Datav indexAsync.indexRole(assignment); } - // Check if ROLE_ASSIGNMENT_AUDITING feature flag is enabled - if (FeatureFlags.ROLE_ASSIGNMENT_AUDITING.enabled()) { - RoleAssignmentHistory audit = new RoleAssignmentHistory(assignment, req, RoleAssignmentHistory.ActionType.ASSIGN); - saveAudit(audit); + // Check if ROLE_ASSIGNMENT_HISTORY feature flag is enabled + if (FeatureFlags.ROLE_ASSIGNMENT_HISTORY.enabled()) { + RoleAssignmentHistory entry = new RoleAssignmentHistory(assignment, req, RoleAssignmentHistory.ActionType.ASSIGN); + saveHistoryEntry(entry); } return assignment; @@ -105,17 +105,17 @@ public RoleAssignment save(RoleAssignment assignment, boolean createIndex, Datav /** * Saves a RoleAssignmentHistory entry to the database. * - * @param audit The RoleAssignmentHistory object to be saved + * @param entry The RoleAssignmentHistory object to be saved * @return The persisted RoleAssignmentHistory object */ - private RoleAssignmentHistory saveAudit(RoleAssignmentHistory audit) { - if (audit.getAuditId() == null) { - em.persist(audit); + private RoleAssignmentHistory saveHistoryEntry(RoleAssignmentHistory entry) { + if (entry.getEntryId() == null) { + em.persist(entry); em.flush(); // Ensure the entity is persisted immediately } else { - audit = em.merge(audit); + entry = em.merge(entry); } - return audit; + return entry; } private IndexResponse indexDefinitionPoint(DvObject definitionPoint) { @@ -182,10 +182,10 @@ public void revoke(RoleAssignment ra, DataverseRequest req) { ra = em.merge(ra); } - // Create audit entry if feature flag is set - if (FeatureFlags.ROLE_ASSIGNMENT_AUDITING.enabled()) { - RoleAssignmentHistory audit = new RoleAssignmentHistory(ra, req, RoleAssignmentHistory.ActionType.REVOKE); - saveAudit(audit); + // Create history entry if feature flag is set + if (FeatureFlags.ROLE_ASSIGNMENT_HISTORY.enabled()) { + RoleAssignmentHistory entry = new RoleAssignmentHistory(ra, req, RoleAssignmentHistory.ActionType.REVOKE); + saveHistoryEntry(entry); } em.remove(ra); @@ -207,10 +207,10 @@ public void revokeAll(RoleAssignee assignee, DataverseRequest req) { ra = em.merge(ra); } - // Create audit entry if feature flag is set - if (FeatureFlags.ROLE_ASSIGNMENT_AUDITING.enabled()) { - RoleAssignmentHistory audit = new RoleAssignmentHistory(ra, req, RoleAssignmentHistory.ActionType.REVOKE); - saveAudit(audit); + // Create history entry if feature flag is set + if (FeatureFlags.ROLE_ASSIGNMENT_HISTORY.enabled()) { + RoleAssignmentHistory entry = new RoleAssignmentHistory(ra, req, RoleAssignmentHistory.ActionType.REVOKE); + saveHistoryEntry(entry); } em.remove(ra); @@ -379,12 +379,12 @@ For a given permission and dataverse Id get all of the roles (built-in or owned * @param definitionPointId The ID of the definition point * @return List of role assignment history entries */ - public List getRoleAssignmentHistory(Long definitionPointId) { - List audits = em.createNamedQuery("RoleAssignmentHistory.findByDefinitionPointId", RoleAssignmentHistory.class) + public List getRoleAssignmentHistory(Long definitionPointId) { + List entries = em.createNamedQuery("RoleAssignmentHistory.findByDefinitionPointId", RoleAssignmentHistory.class) .setParameter("definitionPointId", definitionPointId) .getResultList(); - return processRoleAssignmentAudits(audits, false); + return processRoleAssignmentEntries(entries, false); } /** @@ -393,53 +393,53 @@ public List getRoleAssignmentHistory(Long definition * @param datasetId The ID of the dataset * @return List of role assignment history entries */ - public List getFilesRoleAssignmentHistory(Long datasetId) { - List audits = em.createNamedQuery("RoleAssignmentHistory.findByOwnerId", RoleAssignmentHistory.class) + public List getFilesRoleAssignmentHistory(Long datasetId) { + List entries = em.createNamedQuery("RoleAssignmentHistory.findByOwnerId", RoleAssignmentHistory.class) .setParameter("datasetId", datasetId) .getResultList(); - return processRoleAssignmentAudits(audits, true); + return processRoleAssignmentEntries(entries, true); } /** - * Common method to process role assignment history rows and create consolidated history entries + * Common method to process role assignment history entries and create consolidated history entries * - * @param audits List of role assignment audit records + * @param entries List of role assignment history records * @param combineEntries Whether to combine entries for different files * @return List of role assignment history entries */ - private List processRoleAssignmentAudits(List audits, boolean combineEntries) { - List roleAssignmentHistory = new ArrayList<>(); - Map historyMap = new HashMap<>(); - - // First pass: Create entries from audit records - for (RoleAssignmentHistory audit : audits) { - Long roleAssignmentId = audit.getRoleAssignmentId(); - RoleAssignmentHistoryEntry entry = historyMap.get(roleAssignmentId); - - if (entry == null) { - entry = new RoleAssignmentHistoryEntry(audit.getAssigneeIdentifier(), audit.getRoleAlias(), audit.getDefinitionPointId()); - historyMap.put(roleAssignmentId, entry); + private List processRoleAssignmentEntries(List entries, boolean combineEntries) { + List roleAssignmentHistory = new ArrayList<>(); + Map historyMap = new HashMap<>(); + + // First pass: Create consolidatedEntries from history records + for (RoleAssignmentHistory entry : entries) { + Long roleAssignmentId = entry.getRoleAssignmentId(); + RoleAssignmentHistoryConsolidatedEntry consolidatedEntry = historyMap.get(roleAssignmentId); + + if (consolidatedEntry == null) { + consolidatedEntry = new RoleAssignmentHistoryConsolidatedEntry(entry.getAssigneeIdentifier(), entry.getRoleAlias(), entry.getDefinitionPointId()); + historyMap.put(roleAssignmentId, consolidatedEntry); } - if (audit.getActionType() == RoleAssignmentHistory.ActionType.ASSIGN) { - entry.setAssignedBy(audit.getActionByIdentifier()); - entry.setAssignedAt(audit.getActionTimestamp()); - } else if (audit.getActionType() == RoleAssignmentHistory.ActionType.REVOKE) { - entry.setRevokedBy(audit.getActionByIdentifier()); - entry.setRevokedAt(audit.getActionTimestamp()); + if (entry.getActionType() == RoleAssignmentHistory.ActionType.ASSIGN) { + consolidatedEntry.setAssignedBy(entry.getActionByIdentifier()); + consolidatedEntry.setAssignedAt(entry.getActionTimestamp()); + } else if (entry.getActionType() == RoleAssignmentHistory.ActionType.REVOKE) { + consolidatedEntry.setRevokedBy(entry.getActionByIdentifier()); + consolidatedEntry.setRevokedAt(entry.getActionTimestamp()); } } // Second pass: Combine entries with matching criteria if requested if (combineEntries) { - Map finalHistoryMap = new HashMap<>(); - for (RoleAssignmentHistoryEntry entry : historyMap.values()) { + Map finalHistoryMap = new HashMap<>(); + for (RoleAssignmentHistoryConsolidatedEntry entry : historyMap.values()) { String key = entry.getAssigneeIdentifier() + "|" + entry.getRoleName() + "|" + entry.getAssignedBy() + "|" + entry.getAssignedAt() + "|" + entry.getRevokedBy() + "|" + entry.getRevokedAt(); - RoleAssignmentHistoryEntry existingEntry = finalHistoryMap.get(key); + RoleAssignmentHistoryConsolidatedEntry existingEntry = finalHistoryMap.get(key); if (existingEntry == null) { finalHistoryMap.put(key, entry); } else { @@ -453,14 +453,14 @@ private List processRoleAssignmentAudits(List definitionPointIds; // New field - public RoleAssignmentHistoryEntry(String assigneeIdentifier, String roleName, Long definitionPointId) { + public RoleAssignmentHistoryConsolidatedEntry(String assigneeIdentifier, String roleName, Long definitionPointId) { this.roleName = roleName; this.assigneeIdentifier = assigneeIdentifier; this.definitionPointIds = new ArrayList(); diff --git a/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentHistory.java b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentHistory.java index ebc6d95288a..62bed6b3233 100644 --- a/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentHistory.java +++ b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentHistory.java @@ -99,7 +99,7 @@ public RoleAssignmentHistory(RoleAssignment roleAssignment, DataverseRequest req } // Getters and setters - public Long getAuditId() { + public Long getEntryId() { return auditId; } From ec2042dcddbeeb8f732e51565af375c1c033edb8 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 25 Jul 2025 11:53:18 -0400 Subject: [PATCH 060/634] documentation --- doc/release-notes/11612-RAHistory.md | 17 ++ doc/sphinx-guides/source/api/native-api.rst | 161 ++++++++++++++++++ .../source/installation/config.rst | 3 + .../source/user/dataverse-management.rst | 4 +- 4 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 doc/release-notes/11612-RAHistory.md diff --git a/doc/release-notes/11612-RAHistory.md b/doc/release-notes/11612-RAHistory.md new file mode 100644 index 00000000000..b61fc9d0406 --- /dev/null +++ b/doc/release-notes/11612-RAHistory.md @@ -0,0 +1,17 @@ +# Role Assignment History Tracking + +Dataverse can now track the history of role assignments, allowing administrators to see who assigned or revoked roles, when these actions occurred, and which roles were involved. This feature helps with auditing and understanding permission changes over time. + +## Key components of this feature: + +- **Feature Flag**: The functionality can be enabled/disabled via the `ROLE_ASSIGNMENT_HISTORY` feature flag +- **UI Integration**: New history panels on permission management pages showing the complete history of role assignments/revocations +- **CSV Export**: Administrators can download the role assignment history for a given collection or dataset (or files in a dataset) as a CSV file directly from the new panels +- **API Access**: New API endpoints provide access to role assignment history in both JSON and CSV formats: + - `/api/dataverses/{identifier}/assignments/history` + - `/api/datasets/{identifier}/assignments/history` + - `/api/datasets/{identifier}/files/assignments/history` + +All return JSON by default but will return an internationalized CSV if an `Accept: text/csv` header is adde + +For more information, see #11612 \ No newline at end of file diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 95307f77e48..431c2776a50 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1440,6 +1440,60 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/dataverses/1/templates" --upload-file dataverse-template.json + +Dataverse Role Assignment History +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Get the history of role assignments for a collection. This API call returns a list of role assignments and revocations for the specified dataset. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=1 + + curl -H "X-Dataverse-key:$API_TOKEN" -H "Accept: application/json" "$SERVER_URL/api/dataverses/$ID/assignments/history" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept: application/json" "https://demo.dataverse.org/api/dataverses/3/assignments/history" + +You can also use the collection alias instead of the numeric id: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export DV_ALIAS=dvAlias + + curl -H "X-Dataverse-key:$API_TOKEN" -H "Accept: application/json" "$SERVER_URL/api/dataverses/$DV_ALIAS/assignments/history" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept: application/json" "https://demo.dataverse.org/api/datasets/dvAlias/assignments/history" + +To retrieve the history in CSV format, change the Accept header to "text/csv": + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=3 + + curl -H "X-Dataverse-key:$API_TOKEN" -H "Accept: text/csv" "$SERVER_URL/api/dataverses/$ID/assignments/history" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept: text/csv" "https://demo.dataverse.org/api/dataverses/3/assignments/history" + +Note: This feature requires the "role-assignment-history" feature flag to be enabled (see :ref:`feature-flags`). + Datasets -------- @@ -4009,6 +4063,113 @@ Upon success, the API will return a JSON response with a success message and the The API call will report a 400 (BAD REQUEST) error if any of the files specified do not exist or are not in the latest version of the specified dataset. The ``fileIds`` in the JSON payload should be an array of file IDs that you want to delete from the dataset. +.. _api-dataset-role-assignment-history: + +Dataset Role Assignment History +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Get the history of role assignments for a dataset. This API call returns a list of role assignments and revocations for the specified dataset. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=3 + + curl -H "X-Dataverse-key:$API_TOKEN" -H "Accept: application/json" "$SERVER_URL/api/datasets/$ID/assignments/history" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept: application/json" "https://demo.dataverse.org/api/datasets/3/assignments/history" + +You can also use the persistent identifier instead of the numeric id: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/ABCDEF + + curl -H "X-Dataverse-key:$API_TOKEN" -H "Accept: application/json" "$SERVER_URL/api/datasets/:persistentId/assignments/history?persistentId=$PERSISTENT_IDENTIFIER" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept: application/json" "https://demo.dataverse.org/api/datasets/:persistentId/assignments/history?persistentId=doi:10.5072/FK2/ABCDEF" + +To retrieve the history in CSV format, change the Accept header to "text/csv": + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=3 + + curl -H "X-Dataverse-key:$API_TOKEN" -H "Accept: text/csv" "$SERVER_URL/api/datasets/$ID/assignments/history" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept: text/csv" "https://demo.dataverse.org/api/datasets/3/assignments/history" + +Note: This feature requires the "role-assignment-history" feature flag to be enabled (see :ref:`feature-flags`). + +Dataset Files Role Assignment History +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Get the history of role assignments for the files in a dataset. This API call returns a list of role assignments and revocations for all files in the specified dataset. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=3 + + curl -H "X-Dataverse-key:$API_TOKEN" -H "Accept: application/json" "$SERVER_URL/api/datasets/$ID/files/assignments/history" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept: application/json" "https://demo.dataverse.org/api/datasets/3/files/assignments/history" + +You can also use the persistent identifier instead of the numeric id: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/ABCDEF + + curl -H "X-Dataverse-key:$API_TOKEN" -H "Accept: application/json" "$SERVER_URL/api/datasets/:persistentId/files/assignments/history?persistentId=$PERSISTENT_IDENTIFIER" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept: application/json" "https://demo.dataverse.org/api/datasets/:persistentId/files/assignments/history?persistentId=doi:10.5072/FK2/ABCDEF" + +To retrieve the history in CSV format, change the Accept header to "text/csv": + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=3 + + curl -H "X-Dataverse-key:$API_TOKEN" -H "Accept: text/csv" "$SERVER_URL/api/datasets/files/$ID/assignments/history" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept: text/csv" "https://demo.dataverse.org/api/datasets/3/files/assignments/history" + +Note: This feature requires the "role-assignment-history" feature flag to be enabled (see :ref:`feature-flags`). Files ----- diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 83a6d42c15d..eb177f2f274 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3750,6 +3750,9 @@ please find all known feature flags below. Any of these flags can be activated u * - enable-version-note - Turns on the ability to add/view/edit/delete per-dataset-version notes intended to provide :ref:`provenance` information about why the dataset/version was created. - ``Off`` + * - role-assignment-history + - Turns on tracking/display of role assignments and revocations for collections, datasets, and files + - ``Off`` **Note:** Feature flags can be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_FEATURE_XXX`` (e.g. ``DATAVERSE_FEATURE_API_SESSION_AUTH=1``). These environment variables can be set in your shell before starting Payara. If you are using :doc:`Docker for development `, you can set them in the `docker compose `_ file. diff --git a/doc/sphinx-guides/source/user/dataverse-management.rst b/doc/sphinx-guides/source/user/dataverse-management.rst index d88d0a45e68..ecbafc807a4 100755 --- a/doc/sphinx-guides/source/user/dataverse-management.rst +++ b/doc/sphinx-guides/source/user/dataverse-management.rst @@ -119,7 +119,7 @@ Clicking on Permissions will bring you to this page: |image3| -When you access a Dataverse collection's permissions page, you will see three sections: +When you access a Dataverse collection's permissions page, you will see three or four sections: **Permissions:** Here you can decide the requirements that determine which types of users can add datasets and sub Dataverse collections to your Dataverse collection, and what permissions they'll be granted when they do so. @@ -127,6 +127,8 @@ When you access a Dataverse collection's permissions page, you will see three se **Roles:** Here you can reference a full list of roles that can be assigned to users of your Dataverse collection. Each role lists the permissions that it offers. +**Role Assignment History** If enabled, you'll be able to see the history of when roles have been assigned and revoked and by whom. + Please note that even on a newly created Dataverse collection, you may see user and groups have already been granted role(s) if your installation has ``:InheritParentRoleAssignments`` set. For more on this setting, see the :doc:`/installation/config` section of the Installation Guide. Setting Access Configurations From dfba6a74669fb18fad52e4e14b0a12f63084dc8c Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 25 Jul 2025 13:46:35 -0400 Subject: [PATCH 061/634] change revokeRole to add OnDataverse, add dv roleassignment/history call --- .../edu/harvard/iq/dataverse/api/Dataverses.java | 2 +- .../edu/harvard/iq/dataverse/api/DatasetsIT.java | 2 +- .../java/edu/harvard/iq/dataverse/api/UtilIT.java | 12 +++++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 9dc36b253cc..156f868753c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1965,7 +1965,7 @@ public Response createTemplate(@Context ContainerRequestContext crc, String body @GET @AuthRequired - @Path("{identifier}/permissions/history") + @Path("{identifier}/assignments/history") @Produces({ MediaType.APPLICATION_JSON, "text/csv" }) public Response getRoleAssignmentHistory(@Context ContainerRequestContext crc, @PathParam("identifier") String id, diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index e0d15e13b6d..cc8312b0868 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -2053,7 +2053,7 @@ public void testPrivateUrl() { int roleAssignmentId = (int) roleAssignment.get("id"); logger.info("role assignment id: " + roleAssignmentId); assertEquals(roleAssignmentIdFromCreate, roleAssignmentId); - Response revoke = UtilIT.revokeRole(dataverseAlias, roleAssignmentId, apiToken); + Response revoke = UtilIT.revokeRoleOnDataverse(dataverseAlias, roleAssignmentId, apiToken); revoke.prettyPrint(); assertEquals(OK.getStatusCode(), revoke.getStatusCode()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index eb191c1bad7..fa340600ed2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2575,7 +2575,7 @@ static Response grantRoleOnDataset(String definitionPoint, String role, String r .post("api/datasets/:persistentId/assignments?key=" + apiToken + "&persistentId=" + definitionPoint); } - static Response revokeRole(String definitionPoint, long doomed, String apiToken) { + static Response revokeRoleOnDataverse(String definitionPoint, long doomed, String apiToken) { return given() .header(API_TOKEN_HTTP_HEADER, apiToken) .delete("api/dataverses/" + definitionPoint + "/assignments/" + doomed); @@ -5006,4 +5006,14 @@ public static Response getTemplates(String dataverseAlias, String apiToken) { .header(API_TOKEN_HTTP_HEADER, apiToken) .get("/api/dataverses/" + dataverseAlias + "/templates"); } + + public static Response getDataverseRoleAssignmentHistory(String dataverseAlias, boolean downloadAsCsv, String apiToken) { + RequestSpecification requestSpecification = given() + .header(API_TOKEN_HTTP_HEADER, apiToken); + + requestSpecification = requestSpecification.header("Accept", downloadAsCsv ? "text/csv" : "application/json"); + + return requestSpecification + .get("/api/v1/dataverses/" + dataverseAlias + "/assignments/history"); + } } From bc97830b7afa0b6ef73081a8adf7612d31947eee Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 25 Jul 2025 13:46:46 -0400 Subject: [PATCH 062/634] dataverse-level test --- .../iq/dataverse/api/AbstractApiBean.java | 34 ++-- .../api/RoleAssignmentHistoryIT.java | 146 ++++++++++++++++++ 2 files changed, 170 insertions(+), 10 deletions(-) create mode 100644 src/test/java/edu/harvard/iq/dataverse/api/RoleAssignmentHistoryIT.java diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index fb5868d76b0..59a0a075923 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -697,16 +697,7 @@ protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, Authentic String revokedAtHeader = BundleUtil.getStringFromBundle("dataverse.permissions.history.revokedAt"); // Generate CSV response - StringBuilder csvBuilder = new StringBuilder(); - // Add CSV header with internationalized column names - csvBuilder - .append(definedOn).append(",") - .append(assigneeHeader).append(",") - .append(roleHeader).append(",") - .append(assignedByHeader).append(",") - .append(assignedAtHeader).append(",") - .append(revokedByHeader).append(",") - .append(revokedAtHeader).append("\n"); + StringBuilder csvBuilder = getHistoryCsvHeaderRow(); // Add data rows for (DataverseRoleServiceBean.RoleAssignmentHistoryConsolidatedEntry entry : history) { @@ -771,6 +762,29 @@ protected Response getRoleAssignmentHistoryResponse(DvObject dvObject, Authentic return ok(jsonArray); } + static StringBuilder getHistoryCsvHeaderRow() { + // Reusing strings from history panel + String definedOn = BundleUtil.getStringFromBundle("dataverse.permissions.history.definedOn"); + String assigneeHeader = BundleUtil.getStringFromBundle("dataverse.permissions.history.assignee"); + String roleHeader = BundleUtil.getStringFromBundle("dataverse.permissions.history.role"); + String assignedByHeader = BundleUtil.getStringFromBundle("dataverse.permissions.history.assignedBy"); + String assignedAtHeader = BundleUtil.getStringFromBundle("dataverse.permissions.history.assignedAt"); + String revokedByHeader = BundleUtil.getStringFromBundle("dataverse.permissions.history.revokedBy"); + String revokedAtHeader = BundleUtil.getStringFromBundle("dataverse.permissions.history.revokedAt"); + + // Generate CSV response + StringBuilder csvBuilder = new StringBuilder(); + // Add CSV header with internationalized column names + csvBuilder + .append(definedOn).append(",") + .append(assigneeHeader).append(",") + .append(roleHeader).append(",") + .append(assignedByHeader).append(",") + .append(assignedAtHeader).append(",") + .append(revokedByHeader).append(",") + .append(revokedAtHeader).append("\n"); + return csvBuilder; + } /* =================== *\ * Command Execution * \* =================== */ diff --git a/src/test/java/edu/harvard/iq/dataverse/api/RoleAssignmentHistoryIT.java b/src/test/java/edu/harvard/iq/dataverse/api/RoleAssignmentHistoryIT.java new file mode 100644 index 00000000000..3548cc074f7 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/api/RoleAssignmentHistoryIT.java @@ -0,0 +1,146 @@ +package edu.harvard.iq.dataverse.api; + +import io.restassured.RestAssured; +import io.restassured.path.json.JsonPath; +import io.restassured.response.Response; +import jakarta.json.JsonArray; +import jakarta.json.JsonObject; +import jakarta.json.JsonValue; + +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; + +import static jakarta.ws.rs.core.Response.Status.*; +import static org.hamcrest.CoreMatchers.*; +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import edu.harvard.iq.dataverse.authorization.DataverseRole; +import edu.harvard.iq.dataverse.util.BundleUtil; + +/** + * Integration tests for the Role Assignment History API endpoints. + * + * @author [Your Name] + */ +public class RoleAssignmentHistoryIT { + + private static final Logger logger = Logger.getLogger(RoleAssignmentHistoryIT.class.getCanonicalName()); + + @BeforeAll + public static void setUp() { + RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); + } + + @Test + public void testDataverseRoleAssignmentHistory() { + // Create admin user + Response createAdminUser = UtilIT.createRandomUser(); + String adminUsername = UtilIT.getUsernameFromResponse(createAdminUser); + String adminApiToken = UtilIT.getApiTokenFromResponse(createAdminUser); + UtilIT.setSuperuserStatus(adminUsername, true); + + // Create regular users + Response createUser1 = UtilIT.createRandomUser(); + String username1 = UtilIT.getUsernameFromResponse(createUser1); + + Response createUser2 = UtilIT.createRandomUser(); + String username2 = UtilIT.getUsernameFromResponse(createUser2); + + // Create dataverse + Response createDataverseResponse = UtilIT.createRandomDataverse(adminApiToken); + createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + // Assign roles to users + Response grantContributor = UtilIT.grantRoleOnDataverse(dataverseAlias, DataverseRole.DS_CONTRIBUTOR, "@" + username1, adminApiToken); + grantContributor.then().assertThat().statusCode(OK.getStatusCode()); + + Response grantCurator = UtilIT.grantRoleOnDataverse(dataverseAlias, DataverseRole.CURATOR, "@" + username2, adminApiToken); + grantCurator.then().assertThat().statusCode(OK.getStatusCode()); + + // Revoke role from user1 + grantContributor.prettyPrint(); + String idToDelete = JsonPath.from(grantContributor.getBody().asString()).getString("data.id"); + Response revokeContributor = UtilIT.revokeRoleOnDataverse(dataverseAlias, Long.parseLong(idToDelete), adminApiToken); + revokeContributor.then().assertThat().statusCode(OK.getStatusCode()); + + // Get role assignment history in JSON format + Response historyJson = UtilIT.getDataverseRoleAssignmentHistory(dataverseAlias, false, adminApiToken); + historyJson.then().assertThat().statusCode(OK.getStatusCode()); + + historyJson.prettyPrint(); + + // Verify JSON response structure + List> data = JsonPath.from(historyJson.getBody().asString()).getList("data"); + // History should contain 2 entries + assertTrue(data.size() == 2); + + // Verify the first history entry (the one unrevoked assignment) + Map firstEntry = data.get(0); + // Entry should have definedOn field + assertTrue(firstEntry.containsKey("definedOn")); + assertTrue(firstEntry.containsKey("assigneeIdentifier")); + assertEquals("@" + username2, firstEntry.get("assigneeIdentifier")); + assertTrue(firstEntry.containsKey("roleName")); + assertEquals(DataverseRole.CURATOR, firstEntry.get("roleName")); + assertTrue(firstEntry.containsKey("assignedBy")); + assertEquals("@" + adminUsername, firstEntry.get("assignedBy")); + assertTrue(firstEntry.containsKey("assignedAt")); + assertNotEquals(JsonValue.NULL, firstEntry.get("assignedAt")); + assertTrue(firstEntry.containsKey("revokedBy")); + assertEquals(JsonValue.NULL, firstEntry.get("revokedBy")); + assertTrue(firstEntry.containsKey("revokedAt")); + assertEquals(JsonValue.NULL, firstEntry.get("revokedAt")); + + // Verify the second history entry + Map secondEntry = data.get(0); + // Entry should have definedOn field + assertTrue(secondEntry.containsKey("definedOn")); + assertTrue(secondEntry.containsKey("assigneeIdentifier")); + assertEquals("@" + username1, secondEntry.get("assigneeIdentifier")); + assertTrue(secondEntry.containsKey("roleName")); + assertEquals(DataverseRole.DS_CONTRIBUTOR, secondEntry.get("roleName")); + assertTrue(secondEntry.containsKey("assignedBy")); + assertEquals("@" + adminUsername, secondEntry.get("assignedBy")); + assertTrue(secondEntry.containsKey("assignedAt")); + assertNotEquals(JsonValue.NULL, secondEntry.get("assignedAt")); + assertTrue(secondEntry.containsKey("revokedBy")); + assertEquals("@" + adminUsername, secondEntry.get("revokedBy")); + assertTrue(secondEntry.containsKey("revokedAt")); + assertNotEquals(JsonValue.NULL, secondEntry.get("revokedAt")); + + // Get role assignment history in CSV format + Response historyCsv = UtilIT.getDataverseRoleAssignmentHistory(dataverseAlias, false, adminApiToken); + historyCsv.then().assertThat().statusCode(OK.getStatusCode()); + + // Generate CSV response + StringBuilder csvBuilder = AbstractApiBean.getHistoryCsvHeaderRow(); + // Verify CSV response + String csvBody = historyCsv.getBody().asString(); + assertTrue(csvBody.startsWith(csvBuilder.toString())); + String[] strings = csvBody.split("\n"); + assertTrue(strings[1].contains("@" + username2 + "," + DataverseRole.CURATOR + ",@" + adminUsername)); + assertTrue(strings[2].contains("@" + username2 + "," + DataverseRole.DS_CONTRIBUTOR + ",@" + adminUsername)); + + // Clean up + Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, adminApiToken); + deleteDataverseResponse.then().assertThat().statusCode(OK.getStatusCode()); + + Response deleteUser2Response = UtilIT.deleteUser(username2); + deleteUser2Response.prettyPrint(); + assertEquals(200, deleteUser2Response.getStatusCode()); + + Response deleteUser1Response = UtilIT.deleteUser(username1); + deleteUser1Response.prettyPrint(); + assertEquals(200, deleteUser1Response.getStatusCode()); + + Response deleteAdminUserResponse = UtilIT.deleteUser(adminUsername); + deleteAdminUserResponse.prettyPrint(); + assertEquals(200, deleteAdminUserResponse.getStatusCode()); + } +} + From fd4983c15ba2fb4e5c04e0907f56fb32e1f00f08 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 25 Jul 2025 14:55:07 -0400 Subject: [PATCH 063/634] Change to combine up to the same minute --- .../harvard/iq/dataverse/DataverseRoleServiceBean.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java index 3b058c00c82..ce8d7ea7328 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java @@ -435,10 +435,12 @@ private List processRoleAssignmentEntrie if (combineEntries) { Map finalHistoryMap = new HashMap<>(); for (RoleAssignmentHistoryConsolidatedEntry entry : historyMap.values()) { + // Test for dates that are the same to the minute String key = entry.getAssigneeIdentifier() + "|" + entry.getRoleName() + "|" + - entry.getAssignedBy() + "|" + entry.getAssignedAt() + "|" + - entry.getRevokedBy() + "|" + entry.getRevokedAt(); - + entry.getAssignedBy() + "|" + + (entry.getAssignedAt() != null ? new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm").format(entry.getAssignedAt()) : "null") + "|" + + entry.getRevokedBy() + "|" + + (entry.getRevokedAt() != null ? new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm").format(entry.getRevokedAt()) : "null"); RoleAssignmentHistoryConsolidatedEntry existingEntry = finalHistoryMap.get(key); if (existingEntry == null) { finalHistoryMap.put(key, entry); From 94f4ff23036cf17a96f60dddfe222bc2868cfd1a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 25 Jul 2025 14:55:59 -0400 Subject: [PATCH 064/634] dataset, files ra history tests, fix csv for dataverse test --- .../api/RoleAssignmentHistoryIT.java | 294 ++++++++++++++++-- .../edu/harvard/iq/dataverse/api/UtilIT.java | 20 ++ 2 files changed, 293 insertions(+), 21 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/RoleAssignmentHistoryIT.java b/src/test/java/edu/harvard/iq/dataverse/api/RoleAssignmentHistoryIT.java index 3548cc074f7..97f5e7af544 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/RoleAssignmentHistoryIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/RoleAssignmentHistoryIT.java @@ -27,14 +27,14 @@ * @author [Your Name] */ public class RoleAssignmentHistoryIT { - + private static final Logger logger = Logger.getLogger(RoleAssignmentHistoryIT.class.getCanonicalName()); @BeforeAll public static void setUp() { RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); } - + @Test public void testDataverseRoleAssignmentHistory() { // Create admin user @@ -42,43 +42,43 @@ public void testDataverseRoleAssignmentHistory() { String adminUsername = UtilIT.getUsernameFromResponse(createAdminUser); String adminApiToken = UtilIT.getApiTokenFromResponse(createAdminUser); UtilIT.setSuperuserStatus(adminUsername, true); - + // Create regular users Response createUser1 = UtilIT.createRandomUser(); String username1 = UtilIT.getUsernameFromResponse(createUser1); - + Response createUser2 = UtilIT.createRandomUser(); String username2 = UtilIT.getUsernameFromResponse(createUser2); - + // Create dataverse Response createDataverseResponse = UtilIT.createRandomDataverse(adminApiToken); createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); - + // Assign roles to users Response grantContributor = UtilIT.grantRoleOnDataverse(dataverseAlias, DataverseRole.DS_CONTRIBUTOR, "@" + username1, adminApiToken); grantContributor.then().assertThat().statusCode(OK.getStatusCode()); - + Response grantCurator = UtilIT.grantRoleOnDataverse(dataverseAlias, DataverseRole.CURATOR, "@" + username2, adminApiToken); grantCurator.then().assertThat().statusCode(OK.getStatusCode()); - + // Revoke role from user1 grantContributor.prettyPrint(); String idToDelete = JsonPath.from(grantContributor.getBody().asString()).getString("data.id"); Response revokeContributor = UtilIT.revokeRoleOnDataverse(dataverseAlias, Long.parseLong(idToDelete), adminApiToken); revokeContributor.then().assertThat().statusCode(OK.getStatusCode()); - + // Get role assignment history in JSON format Response historyJson = UtilIT.getDataverseRoleAssignmentHistory(dataverseAlias, false, adminApiToken); historyJson.then().assertThat().statusCode(OK.getStatusCode()); - + historyJson.prettyPrint(); - + // Verify JSON response structure List> data = JsonPath.from(historyJson.getBody().asString()).getList("data"); // History should contain 2 entries assertTrue(data.size() == 2); - + // Verify the first history entry (the one unrevoked assignment) Map firstEntry = data.get(0); // Entry should have definedOn field @@ -95,7 +95,7 @@ public void testDataverseRoleAssignmentHistory() { assertEquals(JsonValue.NULL, firstEntry.get("revokedBy")); assertTrue(firstEntry.containsKey("revokedAt")); assertEquals(JsonValue.NULL, firstEntry.get("revokedAt")); - + // Verify the second history entry Map secondEntry = data.get(0); // Entry should have definedOn field @@ -112,11 +112,11 @@ public void testDataverseRoleAssignmentHistory() { assertEquals("@" + adminUsername, secondEntry.get("revokedBy")); assertTrue(secondEntry.containsKey("revokedAt")); assertNotEquals(JsonValue.NULL, secondEntry.get("revokedAt")); - + // Get role assignment history in CSV format - Response historyCsv = UtilIT.getDataverseRoleAssignmentHistory(dataverseAlias, false, adminApiToken); + Response historyCsv = UtilIT.getDataverseRoleAssignmentHistory(dataverseAlias, true, adminApiToken); historyCsv.then().assertThat().statusCode(OK.getStatusCode()); - + // Generate CSV response StringBuilder csvBuilder = AbstractApiBean.getHistoryCsvHeaderRow(); // Verify CSV response @@ -125,22 +125,274 @@ public void testDataverseRoleAssignmentHistory() { String[] strings = csvBody.split("\n"); assertTrue(strings[1].contains("@" + username2 + "," + DataverseRole.CURATOR + ",@" + adminUsername)); assertTrue(strings[2].contains("@" + username2 + "," + DataverseRole.DS_CONTRIBUTOR + ",@" + adminUsername)); - + // Clean up Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, adminApiToken); deleteDataverseResponse.then().assertThat().statusCode(OK.getStatusCode()); - + Response deleteUser2Response = UtilIT.deleteUser(username2); deleteUser2Response.prettyPrint(); assertEquals(200, deleteUser2Response.getStatusCode()); - + Response deleteUser1Response = UtilIT.deleteUser(username1); deleteUser1Response.prettyPrint(); assertEquals(200, deleteUser1Response.getStatusCode()); - + Response deleteAdminUserResponse = UtilIT.deleteUser(adminUsername); deleteAdminUserResponse.prettyPrint(); assertEquals(200, deleteAdminUserResponse.getStatusCode()); } -} + @Test + public void testDatasetRoleAssignmentHistory() { + // Create admin user + Response createAdminUser = UtilIT.createRandomUser(); + String adminUsername = UtilIT.getUsernameFromResponse(createAdminUser); + String adminApiToken = UtilIT.getApiTokenFromResponse(createAdminUser); + UtilIT.setSuperuserStatus(adminUsername, true); + + // Create regular users + Response createUser1 = UtilIT.createRandomUser(); + String username1 = UtilIT.getUsernameFromResponse(createUser1); + + Response createUser2 = UtilIT.createRandomUser(); + String username2 = UtilIT.getUsernameFromResponse(createUser2); + + // Create dataverse and dataset + Response createDataverseResponse = UtilIT.createRandomDataverse(adminApiToken); + createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, adminApiToken); + createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + + // Assign roles to users on dataset + Response grantEditor = UtilIT.grantRoleOnDataset(datasetId.toString(), DataverseRole.EDITOR, "@" + username1, adminApiToken); + grantEditor.then().assertThat().statusCode(OK.getStatusCode()); + + Response grantFileDownloader = UtilIT.grantRoleOnDataset(datasetId.toString(), DataverseRole.FILE_DOWNLOADER, "@" + username2, adminApiToken); + grantFileDownloader.then().assertThat().statusCode(OK.getStatusCode()); + + // Revoke role from user1 + grantEditor.prettyPrint(); + Long idToDelete = JsonPath.from(grantEditor.getBody().asString()).getLong("data.id"); + Response revokeEditor = UtilIT.revokeRoleOnDataset(datasetId.toString(), idToDelete, adminApiToken); + revokeEditor.then().assertThat().statusCode(OK.getStatusCode()); + + // Get role assignment history in JSON format + Response historyJson = UtilIT.getDatasetRoleAssignmentHistory(datasetId, false, adminApiToken); + historyJson.then().assertThat().statusCode(OK.getStatusCode()); + + historyJson.prettyPrint(); + + // Verify JSON response structure + List> data = JsonPath.from(historyJson.getBody().asString()).getList("data"); + // History should contain 2 entries + assertTrue(data.size() == 2); + + // Verify the first history entry (the one unrevoked assignment) + Map firstEntry = data.get(0); + // Entry should have definedOn field + assertTrue(firstEntry.containsKey("definedOn")); + assertEquals(datasetId.toString(), firstEntry.get("definedOn")); + assertTrue(firstEntry.containsKey("assigneeIdentifier")); + assertEquals("@" + username2, firstEntry.get("assigneeIdentifier")); + assertTrue(firstEntry.containsKey("roleName")); + assertEquals(DataverseRole.FILE_DOWNLOADER, firstEntry.get("roleName")); + assertTrue(firstEntry.containsKey("assignedBy")); + assertEquals("@" + adminUsername, firstEntry.get("assignedBy")); + assertTrue(firstEntry.containsKey("assignedAt")); + assertNotEquals(JsonValue.NULL, firstEntry.get("assignedAt")); + assertTrue(firstEntry.containsKey("revokedBy")); + assertEquals(JsonValue.NULL, firstEntry.get("revokedBy")); + assertTrue(firstEntry.containsKey("revokedAt")); + assertEquals(JsonValue.NULL, firstEntry.get("revokedAt")); + + // Verify the second history entry + Map secondEntry = data.get(0); + // Entry should have definedOn field + assertTrue(secondEntry.containsKey("definedOn")); + assertTrue(secondEntry.containsKey("assigneeIdentifier")); + assertEquals("@" + username1, secondEntry.get("assigneeIdentifier")); + assertTrue(secondEntry.containsKey("roleName")); + assertEquals(DataverseRole.EDITOR, secondEntry.get("roleName")); + assertTrue(secondEntry.containsKey("assignedBy")); + assertEquals("@" + adminUsername, secondEntry.get("assignedBy")); + assertTrue(secondEntry.containsKey("assignedAt")); + assertNotEquals(JsonValue.NULL, secondEntry.get("assignedAt")); + assertTrue(secondEntry.containsKey("revokedBy")); + assertEquals("@" + adminUsername, secondEntry.get("revokedBy")); + assertTrue(secondEntry.containsKey("revokedAt")); + assertNotEquals(JsonValue.NULL, secondEntry.get("revokedAt")); + + // Get role assignment history in CSV format + Response historyCsv = UtilIT.getDatasetRoleAssignmentHistory(datasetId, true, adminApiToken); + historyCsv.then().assertThat().statusCode(OK.getStatusCode()); + + // Generate CSV response + StringBuilder csvBuilder = AbstractApiBean.getHistoryCsvHeaderRow(); + // Verify CSV response + String csvBody = historyCsv.getBody().asString(); + assertTrue(csvBody.startsWith(csvBuilder.toString())); + String[] strings = csvBody.split("\n"); + assertTrue(strings[1].contains("@" + username2 + "," + DataverseRole.FILE_DOWNLOADER + ",@" + adminUsername)); + assertTrue(strings[2].contains("@" + username1 + "," + DataverseRole.EDITOR + ",@" + adminUsername)); + + // Clean up + Response deleteDatasetResponse = UtilIT.destroyDataset(datasetId, adminApiToken); + deleteDatasetResponse.then().assertThat().statusCode(OK.getStatusCode()); + + Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, adminApiToken); + deleteDataverseResponse.then().assertThat().statusCode(OK.getStatusCode()); + + Response deleteUser2Response = UtilIT.deleteUser(username2); + deleteUser2Response.prettyPrint(); + assertEquals(200, deleteUser2Response.getStatusCode()); + + Response deleteUser1Response = UtilIT.deleteUser(username1); + deleteUser1Response.prettyPrint(); + assertEquals(200, deleteUser1Response.getStatusCode()); + + Response deleteAdminUserResponse = UtilIT.deleteUser(adminUsername); + deleteAdminUserResponse.prettyPrint(); + assertEquals(200, deleteAdminUserResponse.getStatusCode()); + } + + /* + * This test is primarily to check the unique functionality for files where grants on multiple files within the same minute will be combined into one entry. + */ + @Test + public void testFileRoleAssignmentHistory() { + // Create admin user + Response createAdminUser = UtilIT.createRandomUser(); + String adminUsername = UtilIT.getUsernameFromResponse(createAdminUser); + String adminApiToken = UtilIT.getApiTokenFromResponse(createAdminUser); + UtilIT.setSuperuserStatus(adminUsername, true); + + // Create regular user + Response createUser1 = UtilIT.createRandomUser(); + String username1 = UtilIT.getUsernameFromResponse(createUser1); + + // Create dataverse and dataset + Response createDataverseResponse = UtilIT.createRandomDataverse(adminApiToken); + createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, adminApiToken); + createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + + // Upload two files + String pathToFile1 = "src/main/webapp/resources/images/dataverseproject.png"; + Response addFile1Response = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile1, adminApiToken); + addFile1Response.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.files[0].label", equalTo("dataverseproject.png")); + Long file1Id = JsonPath.from(addFile1Response.body().asString()).getLong("data.files[0].dataFile.id"); + + String pathToFile2 = "src/main/webapp/resources/images/cc0.png"; + Response addFile2Response = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile2, adminApiToken); + addFile2Response.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.files[0].label", equalTo("cc0.png")); + Long file2Id = JsonPath.from(addFile2Response.body().asString()).getLong("data.files[0].dataFile.id"); + + // Restrict both files + Response restrictFile1Response = UtilIT.restrictFile(file1Id.toString(), true, adminApiToken); + restrictFile1Response.then().assertThat().statusCode(OK.getStatusCode()); + + Response restrictFile2Response = UtilIT.restrictFile(file2Id.toString(), true, adminApiToken); + restrictFile2Response.then().assertThat().statusCode(OK.getStatusCode()); + + // Assign FileDownloader role to both users on both files + String fileIds = file1Id + ", " + file2Id; + Response grantFileDownloader1 = UtilIT.grantFileAccess(file1Id.toString(), "@" + username1, adminApiToken); + grantFileDownloader1.then().assertThat().statusCode(OK.getStatusCode()); + + Response grantFileDownloader2 = UtilIT.grantFileAccess(file2Id.toString(), "@" + username1, adminApiToken); + grantFileDownloader2.then().assertThat().statusCode(OK.getStatusCode()); + + // Get role assignment history in JSON format + Response historyJson = UtilIT.getDatasetFilesRoleAssignmentHistory(datasetId, false, adminApiToken); + historyJson.then().assertThat().statusCode(OK.getStatusCode()); + + historyJson.prettyPrint(); + + // Verify JSON response structure + List> data = JsonPath.from(historyJson.getBody().asString()).getList("data"); + // History should usually contain 1 entry as both files should be listed on one line if the roles were assigned in the same minute + // When that isn't the case, their assignedAt dates must be different. + assertTrue(data.size() > 0); + + if (data.size() == 1) { + // Verify the first history entry (the one unrevoked assignment) + Map firstEntry = data.get(0); + // Entry should have definedOn field with both file IDs + assertTrue(firstEntry.containsKey("definedOn")); + assertEquals(fileIds, firstEntry.get("definedOn")); + assertTrue(firstEntry.containsKey("assigneeIdentifier")); + assertEquals("@" + username1, firstEntry.get("assigneeIdentifier")); + assertTrue(firstEntry.containsKey("roleName")); + assertEquals(DataverseRole.FILE_DOWNLOADER, firstEntry.get("roleName")); + assertTrue(firstEntry.containsKey("assignedBy")); + assertEquals("@" + adminUsername, firstEntry.get("assignedBy")); + assertTrue(firstEntry.containsKey("assignedAt")); + assertNotEquals(JsonValue.NULL, firstEntry.get("assignedAt")); + assertTrue(firstEntry.containsKey("revokedBy")); + assertEquals(JsonValue.NULL, firstEntry.get("revokedBy")); + assertTrue(firstEntry.containsKey("revokedAt")); + assertEquals(JsonValue.NULL, firstEntry.get("revokedAt")); + } else { + Map firstEntry = data.get(0); + // Entry should have definedOn field with the first file ID + assertTrue(firstEntry.containsKey("definedOn")); + assertEquals(file1Id.toString(), firstEntry.get("definedOn")); + assertTrue(firstEntry.containsKey("assigneeIdentifier")); + assertEquals("@" + username1, firstEntry.get("assigneeIdentifier")); + assertTrue(firstEntry.containsKey("roleName")); + assertEquals(DataverseRole.FILE_DOWNLOADER, firstEntry.get("roleName")); + assertTrue(firstEntry.containsKey("assignedBy")); + assertEquals("@" + adminUsername, firstEntry.get("assignedBy")); + assertTrue(firstEntry.containsKey("assignedAt")); + assertNotEquals(JsonValue.NULL, firstEntry.get("assignedAt")); + assertTrue(firstEntry.containsKey("revokedBy")); + assertEquals(JsonValue.NULL, firstEntry.get("revokedBy")); + assertTrue(firstEntry.containsKey("revokedAt")); + assertEquals(JsonValue.NULL, firstEntry.get("revokedAt")); + Map secondEntry = data.get(1); + // Entry should have definedOn field with the second file ID + assertTrue(secondEntry.containsKey("definedOn")); + assertEquals(file2Id.toString(), secondEntry.get("definedOn")); + assertTrue(secondEntry.containsKey("assigneeIdentifier")); + assertEquals("@" + username1, secondEntry.get("assigneeIdentifier")); + assertTrue(secondEntry.containsKey("roleName")); + assertEquals(DataverseRole.FILE_DOWNLOADER, secondEntry.get("roleName")); + assertTrue(secondEntry.containsKey("assignedBy")); + assertEquals("@" + adminUsername, secondEntry.get("assignedBy")); + assertTrue(secondEntry.containsKey("assignedAt")); + assertNotEquals(JsonValue.NULL, secondEntry.get("assignedAt")); + assertTrue(secondEntry.containsKey("revokedBy")); + assertEquals(JsonValue.NULL, secondEntry.get("revokedBy")); + assertTrue(secondEntry.containsKey("revokedAt")); + assertEquals(JsonValue.NULL, firstEntry.get("revokedAt")); + // For two lines the assignedAt dates should be different + assertNotEquals(firstEntry.get("assignedAt"), secondEntry.get("assignedAt")); + } + + // Clean up + Response deleteDatasetResponse = UtilIT.destroyDataset(datasetId, adminApiToken); + deleteDatasetResponse.then().assertThat().statusCode(OK.getStatusCode()); + + Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, adminApiToken); + deleteDataverseResponse.then().assertThat().statusCode(OK.getStatusCode()); + + Response deleteUser1Response = UtilIT.deleteUser(username1); + deleteUser1Response.prettyPrint(); + assertEquals(200, deleteUser1Response.getStatusCode()); + + Response deleteAdminUserResponse = UtilIT.deleteUser(adminUsername); + deleteAdminUserResponse.prettyPrint(); + assertEquals(200, deleteAdminUserResponse.getStatusCode()); + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index fa340600ed2..167f9ea6c44 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -5016,4 +5016,24 @@ public static Response getDataverseRoleAssignmentHistory(String dataverseAlias, return requestSpecification .get("/api/v1/dataverses/" + dataverseAlias + "/assignments/history"); } + + public static Response getDatasetRoleAssignmentHistory(Integer datasetId, boolean downloadAsCsv, String apiToken) { + RequestSpecification requestSpecification = given() + .header(API_TOKEN_HTTP_HEADER, apiToken); + + requestSpecification = requestSpecification.header("Accept", downloadAsCsv ? "text/csv" : "application/json"); + + return requestSpecification + .get("/api/v1/datasets/" + datasetId + "/assignments/history"); + } + + public static Response getDatasetFilesRoleAssignmentHistory(Integer datasetId, boolean downloadAsCsv, String apiToken) { + RequestSpecification requestSpecification = given() + .header(API_TOKEN_HTTP_HEADER, apiToken); + + requestSpecification = requestSpecification.header("Accept", downloadAsCsv ? "text/csv" : "application/json"); + + return requestSpecification + .get("/api/v1/datasets/" + datasetId + "/files/assignments/history"); + } } From ceaf8551638976621d98bafb5cfcd94a1ba41f19 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 25 Jul 2025 16:27:48 -0400 Subject: [PATCH 065/634] local tests --- .../DataverseRoleServiceBeanTest.java | 311 ++++++++++++++++++ 1 file changed, 311 insertions(+) create mode 100644 src/test/java/edu/harvard/iq/dataverse/DataverseRoleServiceBeanTest.java diff --git a/src/test/java/edu/harvard/iq/dataverse/DataverseRoleServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/DataverseRoleServiceBeanTest.java new file mode 100644 index 00000000000..95a278fd7c7 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/DataverseRoleServiceBeanTest.java @@ -0,0 +1,311 @@ +package edu.harvard.iq.dataverse; + +import edu.harvard.iq.dataverse.DataverseRoleServiceBean.RoleAssignmentHistoryConsolidatedEntry; +import edu.harvard.iq.dataverse.RoleAssignmentHistory; +import edu.harvard.iq.dataverse.authorization.DataverseRole; +import edu.harvard.iq.dataverse.authorization.RoleAssignee; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import jakarta.persistence.EntityManager; +import jakarta.persistence.TypedQuery; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +public class DataverseRoleServiceBeanTest { + + @InjectMocks + private DataverseRoleServiceBean dataverseRoleServiceBean; + + @Mock + private EntityManager em; + + private Method processRoleAssignmentEntriesMethod; + + @BeforeEach + public void setUp() throws Exception { + // Get the private method using reflection + processRoleAssignmentEntriesMethod = DataverseRoleServiceBean.class.getDeclaredMethod( + "processRoleAssignmentEntries", List.class, boolean.class); + processRoleAssignmentEntriesMethod.setAccessible(true); + } + + @Test + public void testProcessRoleAssignmentEntries_WithoutCombining() throws Exception { + // Create test data + List entries = new ArrayList<>(); + + // Create two entries for the same role assignment (assign and revoke) + RoleAssignmentHistory entry1 = createRoleAssignmentHistoryEntry( + 1L, "user1", "role1", 101L, "admin1", + new Date(), RoleAssignmentHistory.ActionType.ASSIGN); + + RoleAssignmentHistory entry2 = createRoleAssignmentHistoryEntry( + 1L, "user1", "role1", 101L, "admin2", + new Date(), RoleAssignmentHistory.ActionType.REVOKE); + + // Create another role assignment for a different user + RoleAssignmentHistory entry3 = createRoleAssignmentHistoryEntry( + 2L, "user2", "role2", 102L, "admin1", + new Date(), RoleAssignmentHistory.ActionType.ASSIGN); + + entries.add(entry1); + entries.add(entry2); + entries.add(entry3); + + // Call the private method + @SuppressWarnings("unchecked") + List result = (List) processRoleAssignmentEntriesMethod.invoke( + dataverseRoleServiceBean, entries, false); + + // Verify results + assertEquals(2, result.size(), "Should have 2 consolidated entries"); + + // Find the entry for user1 - should always be second since it it revoked and the user2 role is not. + RoleAssignmentHistoryConsolidatedEntry user1Entry = result.get(1); + + assertNotNull(user1Entry, "Should have an entry for user1"); + assertEquals("role1", user1Entry.getRoleName()); + assertEquals("admin1", user1Entry.getAssignedBy()); + assertEquals("admin2", user1Entry.getRevokedBy()); + assertNotNull(user1Entry.getAssignedAt()); + assertNotNull(user1Entry.getRevokedAt()); + assertEquals(1, user1Entry.getDefinitionPointIds().size()); + assertEquals(101L, user1Entry.getDefinitionPointIds().get(0)); + + // Find the entry for user2 - always fist since it is not revoked + RoleAssignmentHistoryConsolidatedEntry user2Entry = result.get(0); + + assertNotNull(user2Entry, "Should have an entry for user2"); + assertEquals("role2", user2Entry.getRoleName()); + assertEquals("admin1", user2Entry.getAssignedBy()); + assertNull(user2Entry.getRevokedBy()); + assertNotNull(user2Entry.getAssignedAt()); + assertNull(user2Entry.getRevokedAt()); + assertEquals(1, user2Entry.getDefinitionPointIds().size()); + assertEquals(102L, user2Entry.getDefinitionPointIds().get(0)); + } + + @Test + public void testProcessRoleAssignmentEntries_WithCombining() throws Exception { + // Create test data + List entries = new ArrayList<>(); + + // Create entries for the same minute but different definition points + Date baseTime = new Date(); + Calendar cal = Calendar.getInstance(); + cal.setTime(baseTime); + + // First role assignment + RoleAssignmentHistory entry1 = createRoleAssignmentHistoryEntry( + 1L, "user1", "role1", 101L, "admin1", + baseTime, RoleAssignmentHistory.ActionType.ASSIGN); + + // Second role assignment with same minute + RoleAssignmentHistory entry2 = createRoleAssignmentHistoryEntry( + 2L, "user1", "role1", 102L, "admin1", + baseTime, RoleAssignmentHistory.ActionType.ASSIGN); + + // Third role assignment with different minute + cal.add(Calendar.MINUTE, 1); + Date laterTime = cal.getTime(); + RoleAssignmentHistory entry3 = createRoleAssignmentHistoryEntry( + 3L, "user1", "role1", 103L, "admin1", + laterTime, RoleAssignmentHistory.ActionType.ASSIGN); + + // Fourth role assignment with different role + RoleAssignmentHistory entry4 = createRoleAssignmentHistoryEntry( + 4L, "user1", "role2", 104L, "admin1", + baseTime, RoleAssignmentHistory.ActionType.ASSIGN); + + entries.add(entry1); + entries.add(entry2); + entries.add(entry3); + entries.add(entry4); + + // Call the private method + @SuppressWarnings("unchecked") + List result = (List) processRoleAssignmentEntriesMethod.invoke( + dataverseRoleServiceBean, entries, true); + + // Verify results + assertEquals(3, result.size(), "Should have 3 consolidated entries"); + + // Find the entry for user1 with role1 and baseTime + RoleAssignmentHistoryConsolidatedEntry combinedEntry = result.stream() + .filter(e -> e.getAssigneeIdentifier().equals("user1") && + e.getRoleName().equals("role1") && + e.getDefinitionPointIds().contains(101L)) + .findFirst() + .orElse(null); + + assertNotNull(combinedEntry, "Should have a combined entry for user1 with role1"); + assertEquals(2, combinedEntry.getDefinitionPointIds().size(), + "Combined entry should have 2 definition points"); + assertTrue(combinedEntry.getDefinitionPointIds().contains(101L), + "Combined entry should contain definition point 101"); + assertTrue(combinedEntry.getDefinitionPointIds().contains(102L), + "Combined entry should contain definition point 102"); + + // Find the entry for user1 with role1 and laterTime + RoleAssignmentHistoryConsolidatedEntry laterEntry = result.stream() + .filter(e -> e.getAssigneeIdentifier().equals("user1") && + e.getRoleName().equals("role1") && + !e.getDefinitionPointIds().contains(101L)) + .findFirst() + .orElse(null); + + assertNotNull(laterEntry, "Should have an entry for user1 with role1 at later time"); + assertEquals(1, laterEntry.getDefinitionPointIds().size()); + assertEquals(103L, laterEntry.getDefinitionPointIds().get(0)); + + // Find the entry for user1 with role2 + RoleAssignmentHistoryConsolidatedEntry role2Entry = result.stream() + .filter(e -> e.getAssigneeIdentifier().equals("user1") && + e.getRoleName().equals("role2")) + .findFirst() + .orElse(null); + + assertNotNull(role2Entry, "Should have an entry for user1 with role2"); + assertEquals(1, role2Entry.getDefinitionPointIds().size()); + assertEquals(104L, role2Entry.getDefinitionPointIds().get(0)); + } + + @Test + public void testProcessRoleAssignmentEntries_WithAssignAndRevokeAndCombining() throws Exception { + // Create test data + List entries = new ArrayList<>(); + + // Create base time + Date baseTime = new Date(); + Calendar cal = Calendar.getInstance(); + cal.setTime(baseTime); + + // Create revoke time (5 minutes later) + cal.add(Calendar.MINUTE, 5); + Date revokeTime = cal.getTime(); + + // First role assignment and revoke + RoleAssignmentHistory entry1 = createRoleAssignmentHistoryEntry( + 1L, "user1", "role1", 101L, "admin1", + baseTime, RoleAssignmentHistory.ActionType.ASSIGN); + + RoleAssignmentHistory entry2 = createRoleAssignmentHistoryEntry( + 1L, "user1", "role1", 101L, "admin2", + revokeTime, RoleAssignmentHistory.ActionType.REVOKE); + + // Second role assignment and revoke (same times) + RoleAssignmentHistory entry3 = createRoleAssignmentHistoryEntry( + 2L, "user1", "role1", 102L, "admin1", + baseTime, RoleAssignmentHistory.ActionType.ASSIGN); + + RoleAssignmentHistory entry4 = createRoleAssignmentHistoryEntry( + 2L, "user1", "role1", 102L, "admin2", + revokeTime, RoleAssignmentHistory.ActionType.REVOKE); + + // Third role assignment with different revoke time + cal.add(Calendar.MINUTE, 1); + Date laterRevokeTime = cal.getTime(); + + RoleAssignmentHistory entry5 = createRoleAssignmentHistoryEntry( + 3L, "user1", "role1", 103L, "admin1", + baseTime, RoleAssignmentHistory.ActionType.ASSIGN); + + RoleAssignmentHistory entry6 = createRoleAssignmentHistoryEntry( + 3L, "user1", "role1", 103L, "admin2", + laterRevokeTime, RoleAssignmentHistory.ActionType.REVOKE); + + entries.add(entry1); + entries.add(entry2); + entries.add(entry3); + entries.add(entry4); + entries.add(entry5); + entries.add(entry6); + + // Call the private method + @SuppressWarnings("unchecked") + List result = (List) processRoleAssignmentEntriesMethod.invoke( + dataverseRoleServiceBean, entries, true); + + // Verify results + assertEquals(2, result.size(), "Should have 2 consolidated entries"); + + // Find the entry for user1 with role1 and baseTime - oldest == last + RoleAssignmentHistoryConsolidatedEntry combinedEntry = result.get(1); + + assertNotNull(combinedEntry, "Should have a combined entry for user1 with role1"); + assertEquals(2, combinedEntry.getDefinitionPointIds().size(), + "Combined entry should have 2 definition points"); + assertTrue(combinedEntry.getDefinitionPointIds().contains(101L), + "Combined entry should contain definition point 101"); + assertTrue(combinedEntry.getDefinitionPointIds().contains(102L), + "Combined entry should contain definition point 102"); + + // Find the entry for user1 with role1 and laterTime + RoleAssignmentHistoryConsolidatedEntry laterEntry = result.stream() + .filter(e -> e.getAssigneeIdentifier().equals("user1") && + e.getRoleName().equals("role1") && + !e.getDefinitionPointIds().contains(101L)) + .findFirst() + .orElse(null); + + assertNotNull(laterEntry, "Should have an entry for user1 with role1 at later time"); + assertEquals(1, laterEntry.getDefinitionPointIds().size()); + assertEquals(103L, laterEntry.getDefinitionPointIds().get(0)); + + } + + private RoleAssignmentHistory createRoleAssignmentHistoryEntry( + Long roleAssignmentId, String assigneeIdentifier, String roleAlias, + Long definitionPointId, String actionByIdentifier, + Date actionTimestamp, RoleAssignmentHistory.ActionType actionType) { + + // Create a DataverseRole + DataverseRole role = new DataverseRole(); + role.setAlias(roleAlias); + role.setId(1L); // Arbitrary ID for testing + + // Create a DvObject for definition point + DvObject dvObject = new Dataverse(); + dvObject.setId(definitionPointId); + + // Create a RoleAssignee (using a mock since it's an interface) + RoleAssignee assignee = mock(RoleAssignee.class); + when(assignee.getIdentifier()).thenReturn(assigneeIdentifier); + + // Create the RoleAssignment + RoleAssignment roleAssignment = new RoleAssignment(role, assignee, dvObject, null); + // Set the ID using reflection since it's normally set by the persistence layer + try { + java.lang.reflect.Field idField = RoleAssignment.class.getDeclaredField("id"); + idField.setAccessible(true); + idField.set(roleAssignment, roleAssignmentId); + } catch (Exception e) { + throw new RuntimeException("Failed to set RoleAssignment ID", e); + } + + // Create the RoleAssignmentHistory entry + RoleAssignmentHistory history = new RoleAssignmentHistory(); + history.setRoleAssignmentId(roleAssignmentId); + history.setAssigneeIdentifier(assigneeIdentifier); + history.setRoleAlias(roleAlias); + history.setDefinitionPointId(definitionPointId); + history.setActionByIdentifier(actionByIdentifier); + history.setActionTimestamp(actionTimestamp); + history.setActionType(actionType); + + return history; + } +} \ No newline at end of file From 8ad46ff58f5738f6b824181201957fa1a2772222 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 25 Jul 2025 16:28:04 -0400 Subject: [PATCH 066/634] IT tests (for use when the feature is on) --- .../harvard/iq/dataverse/api/RoleAssignmentHistoryIT.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/RoleAssignmentHistoryIT.java b/src/test/java/edu/harvard/iq/dataverse/api/RoleAssignmentHistoryIT.java index 97f5e7af544..3e105ac9976 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/RoleAssignmentHistoryIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/RoleAssignmentHistoryIT.java @@ -3,8 +3,6 @@ import io.restassured.RestAssured; import io.restassured.path.json.JsonPath; import io.restassured.response.Response; -import jakarta.json.JsonArray; -import jakarta.json.JsonObject; import jakarta.json.JsonValue; import java.util.List; @@ -19,12 +17,12 @@ import org.junit.jupiter.api.Test; import edu.harvard.iq.dataverse.authorization.DataverseRole; -import edu.harvard.iq.dataverse.util.BundleUtil; /** - * Integration tests for the Role Assignment History API endpoints. + * Integration tests for the Role Assignment History API endpoints. + * + * Note: These tests require the role-assignment-history FeatureFlag to be true and are not run in normal builds * - * @author [Your Name] */ public class RoleAssignmentHistoryIT { From 77972b2238d79e6e41302ddb864484a158f2c1d3 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 25 Jul 2025 16:28:25 -0400 Subject: [PATCH 067/634] doc updates with info on response formats --- doc/sphinx-guides/source/api/native-api.rst | 40 +++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 431c2776a50..e81a5a147db 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1476,6 +1476,22 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept: application/json" "https://demo.dataverse.org/api/datasets/dvAlias/assignments/history" +The response is a JSON array of role assignment history entries with the following structure for each entry: + +.. code-block:: json + + { + "definedOn": "1", + "assigneeIdentifier": "@user1", + "roleName": "Admin", + "assignedBy": "@dataverseAdmin", + "assignedAt": "2023-01-01T12:00:00Z", + "revokedBy": null, + "revokedAt": null + } + +For revoked assignments, the "revokedBy" and "revokedAt" fields will contain values instead of null. + To retrieve the history in CSV format, change the Accept header to "text/csv": .. code-block:: bash @@ -1492,6 +1508,8 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept: text/csv" "https://demo.dataverse.org/api/dataverses/3/assignments/history" +The CSV response has column headers mirroring the json entries. They are internationalized (when internationalization is configured). + Note: This feature requires the "role-assignment-history" feature flag to be enabled (see :ref:`feature-flags`). Datasets @@ -4100,6 +4118,22 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept: application/json" "https://demo.dataverse.org/api/datasets/:persistentId/assignments/history?persistentId=doi:10.5072/FK2/ABCDEF" +The response is a JSON array of role assignment history entries with the following structure for each entry: + +.. code-block:: json + + { + "definedOn": "3", + "assigneeIdentifier": "@user1", + "roleName": "Admin", + "assignedBy": "@dataverseAdmin", + "assignedAt": "2023-01-01T12:00:00Z", + "revokedBy": null, + "revokedAt": null + } + +For revoked assignments, the "revokedBy" and "revokedAt" fields will contain values instead of null. + To retrieve the history in CSV format, change the Accept header to "text/csv": .. code-block:: bash @@ -4116,6 +4150,8 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept: text/csv" "https://demo.dataverse.org/api/datasets/3/assignments/history" +The CSV response has column headers mirroring the json entries. They are internationalized (when internationalization is configured). + Note: This feature requires the "role-assignment-history" feature flag to be enabled (see :ref:`feature-flags`). Dataset Files Role Assignment History @@ -4153,6 +4189,8 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept: application/json" "https://demo.dataverse.org/api/datasets/:persistentId/files/assignments/history?persistentId=doi:10.5072/FK2/ABCDEF" +The JSON response for this call is the same as for the /api/datasets/{id}/assignments/history call above with the exception that definedOn will be a comma separated list of one or more file ids. + To retrieve the history in CSV format, change the Accept header to "text/csv": .. code-block:: bash @@ -4169,6 +4207,8 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept: text/csv" "https://demo.dataverse.org/api/datasets/3/files/assignments/history" +The CSV response for this call is the same as for the /api/datasets/{id}/assignments/history call above with the exception that definedOn will be a comma separated list of one or more file ids. + Note: This feature requires the "role-assignment-history" feature flag to be enabled (see :ref:`feature-flags`). Files From a348b98bac4ac871f696954b929fdecd519f0444 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 25 Jul 2025 18:13:04 -0400 Subject: [PATCH 068/634] change table, get rid of audit word --- .../iq/dataverse/RoleAssignmentHistory.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentHistory.java b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentHistory.java index 62bed6b3233..a5f1666794a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentHistory.java +++ b/src/main/java/edu/harvard/iq/dataverse/RoleAssignmentHistory.java @@ -20,15 +20,15 @@ import edu.harvard.iq.dataverse.engine.command.DataverseRequest; @Entity -@Table(name = "role_assignment_audit", indexes = { - @Index(name = "idx_raa_role_assignment_id", columnList = "role_assignment_id"), - @Index(name = "idx_raa_action_type", columnList = "action_type"), - @Index(name = "idx_raa_action_timestamp", columnList = "action_timestamp"), - @Index(name = "idx_raa_action_by_identifier", columnList = "action_by_identifier"), - @Index(name = "idx_raa_assignee_identifier", columnList = "assignee_identifier"), - @Index(name = "idx_raa_role_id", columnList = "role_id"), - @Index(name = "idx_raa_definition_point_id", columnList = "definition_point_id") -}) +@Table(name = "roleassignmenthistory", indexes = { + @Index(columnList = "role_assignment_id"), + @Index(columnList = "action_type"), + @Index(columnList = "action_timestamp"), + @Index(columnList = "action_by_identifier"), + @Index(columnList = "assignee_identifier"), + @Index(columnList = "role_id"), + @Index(columnList = "definition_point_id") + }) @NamedQueries({ @NamedQuery(name = "RoleAssignmentHistory.findByDefinitionPointId", query = "SELECT ra FROM RoleAssignmentHistory ra WHERE ra.definitionPointId = :definitionPointId ORDER BY ra.roleAssignmentId, ra.actionTimestamp DESC"), @@ -41,8 +41,8 @@ public class RoleAssignmentHistory implements Serializable { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) - @Column(name = "audit_id") - private Long auditId; + @Column(name = "entry_id") + private Long entry_id; @Column(name = "role_assignment_id") private Long roleAssignmentId; @@ -100,11 +100,11 @@ public RoleAssignmentHistory(RoleAssignment roleAssignment, DataverseRequest req // Getters and setters public Long getEntryId() { - return auditId; + return entry_id; } - public void setAuditId(Long auditId) { - this.auditId = auditId; + public void setEntryId(Long entryId) { + this.entry_id = entryId; } public Long getRoleAssignmentId() { From a1a85b4a3ce02c890e9eabe38e331135f00df5cd Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 25 Jul 2025 18:26:56 -0400 Subject: [PATCH 069/634] update flag name in xhtml --- src/main/webapp/permissions-manage-files.xhtml | 2 +- src/main/webapp/permissions-manage.xhtml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index 80e0a4eb1d3..5f450c4ae63 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -221,7 +221,7 @@
-
+
#{bundle['dataverse.permissions.history']} #{bundle['dataverse.permissions.history.description']}
diff --git a/src/main/webapp/permissions-manage.xhtml b/src/main/webapp/permissions-manage.xhtml index 354b963be6c..6289089c912 100644 --- a/src/main/webapp/permissions-manage.xhtml +++ b/src/main/webapp/permissions-manage.xhtml @@ -198,7 +198,7 @@
-
+
#{bundle['dataverse.permissions.history']} #{bundle['dataverse.permissions.history.description']}
From 8a8ebe24c9539a615a11842a132cd89f29ffc9b3 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 5 Aug 2025 14:07:44 -0400 Subject: [PATCH 070/634] feature to extend the get storage driver API --- .../11695-change-api-get-storage-driver.md | 10 ++++++ doc/sphinx-guides/source/api/changelog.rst | 1 + .../edu/harvard/iq/dataverse/api/Admin.java | 3 +- .../harvard/iq/dataverse/api/Datasets.java | 2 +- .../iq/dataverse/util/json/JsonPrinter.java | 10 ++++++ .../harvard/iq/dataverse/api/DatasetsIT.java | 31 +++++++++++++++++++ .../harvard/iq/dataverse/api/S3AccessIT.java | 8 ++--- .../edu/harvard/iq/dataverse/api/UtilIT.java | 6 ++++ 8 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 doc/release-notes/11695-change-api-get-storage-driver.md diff --git a/doc/release-notes/11695-change-api-get-storage-driver.md b/doc/release-notes/11695-change-api-get-storage-driver.md new file mode 100644 index 00000000000..3680da4b72c --- /dev/null +++ b/doc/release-notes/11695-change-api-get-storage-driver.md @@ -0,0 +1,10 @@ +## Get Dataset/Dataverse Storage Driver API + +### Changed Json response - breaking change! + +The API for getting the Storage Driver info has been changed/extended. +/api/datasets/{identifier}/storageDriver +/api/admin/dataverse/{dataverse-alias}/storageDriver +changed "message" to "name" and added "type" and "label" + +See also [the guides](https://dataverse-guide--11664.org.readthedocs.build/en/11664/api/native-api.html#configure-a-dataset-to-store-all-new-files-in-a-specific-file-store), #11695, and #11664. diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 16157459220..e8354a2db9f 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -13,6 +13,7 @@ v6.8 - For POST /api/files/{id}/metadata passing an empty string ("description":"") or array ("categories":[]) will no longer be ignored. Empty fields will now clear out the values in the file's metadata. To ignore the fields simply do not include them in the JSON string. - For PUT /api/datasets/{id}/editMetadata the query parameter "sourceInternalVersionNumber" has been removed and replaced with "sourceLastUpdateTime" to verify that the data being edited hasn't been modified and isn't stale. - For GET /api/dataverses/$dataverse-alias/links the Json response has changed breaking the backward compatibility of the API. +- For GET /api/admin/dataverse/{dataverse-alias}/storageDriver and /api/datasets/{identifier}/storageDriver the driver name is no longer returned in data.message. This value is now returned in data.name. v6.7 ---- diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index ac52b5d9fbf..1d726830c31 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -20,6 +20,7 @@ import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.cache.CacheFactoryBean; +import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import edu.harvard.iq.dataverse.validation.EMailValidator; import edu.harvard.iq.dataverse.EjbDataverseEngine; @@ -2196,7 +2197,7 @@ public Response getStorageDriver(@Context ContainerRequestContext crc, @PathPara return wr.getResponse(); } //Note that this returns what's set directly on this dataverse. If null/DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER, the user would have to recurse the chain of parents to find the effective storageDriver - return ok(dataverse.getStorageDriverId()); + return ok(JsonPrinter.jsonStorageDriver(dataverse.getStorageDriverId())); } @PUT diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 3db5e04b4ab..6c0f1460a74 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3701,7 +3701,7 @@ public Response getFileStore(@Context ContainerRequestContext crc, @PathParam("i return error(Response.Status.NOT_FOUND, "No such dataset"); } - return response(req -> ok(dataset.getEffectiveStorageDriverId()), getRequestUser(crc)); + return ok(JsonPrinter.jsonStorageDriver(dataset.getStorageDriverId())); } @PUT diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 592a893083c..a7bf0d637fd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -39,6 +39,7 @@ import edu.harvard.iq.dataverse.workflow.Workflow; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepData; +import java.io.IOException; import java.util.*; import jakarta.json.Json; import jakarta.json.JsonArrayBuilder; @@ -1610,4 +1611,13 @@ public static JsonArrayBuilder jsonTemplateInstructions(Map temp return jsonArrayBuilder; } + + public static JsonObjectBuilder jsonStorageDriver(String storageDriverId) { + JsonObjectBuilder jsonObjectBuilder = new NullSafeJsonBuilder(); + jsonObjectBuilder.add("name", storageDriverId); + jsonObjectBuilder.add("type", DataAccess.getDriverType(storageDriverId)); + jsonObjectBuilder.add("label", DataAccess.getStorageDriverLabelFor(storageDriverId)); + + return jsonObjectBuilder; + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index f3a83c74224..2ad726aaecf 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -5982,6 +5982,37 @@ public void testGetGlobusUploadParameters() { GlobusOverlayAccessIOTest.tearDown(); } + @Test + public void testSetGetDatasetStorageDriver() { + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + String username = UtilIT.getUsernameFromResponse(createUser); + Response makeSuperUser = UtilIT.makeSuperUser(username); + assertEquals(200, makeSuperUser.getStatusCode()); + Response createDataverse = UtilIT.createRandomDataverse(apiToken); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + + Response storageDrivers = UtilIT.listStorageDrivers(apiToken); + storageDrivers.prettyPrint(); + JsonObject data = JsonUtil.getJsonObject(storageDrivers.getBody().asString()); + String first = data.getJsonObject("data").keySet().iterator().next(); + String name = data.getJsonObject("data").getString(first); + + Response setDriver = UtilIT.setDatasetStorageDriver(datasetId, first, apiToken); + setDriver.prettyPrint(); + assertEquals(200, setDriver.getStatusCode()); + Response getDriver = UtilIT.getDatasetStorageDriver(datasetId, apiToken); + getDriver.prettyPrint(); + assertEquals(200, getDriver.getStatusCode()); + getDriver.then().assertThat() + .body("data.name", CoreMatchers.equalTo(name)) + .body("data.type", CoreMatchers.notNullValue()) + .statusCode(OK.getStatusCode()); + } + @Test public void testGetCanDownloadAtLeastOneFile() { Response createUserResponse = UtilIT.createRandomUser(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java index 137c2ef4c7b..d5fecb32937 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java @@ -142,7 +142,7 @@ public void testNonDirectUpload() { Response originalStorageDriver = UtilIT.getStorageDriver(dataverseAlias, superuserApiToken); originalStorageDriver.prettyPrint(); originalStorageDriver.then().assertThat() - .body("data.message", equalTo("undefined")) + .body("data.name", equalTo("undefined")) .statusCode(200); Response setStorageDriverToS3 = UtilIT.setStorageDriver(dataverseAlias, driverLabel, superuserApiToken); @@ -273,7 +273,7 @@ public void testDirectUpload() { Response originalStorageDriver = UtilIT.getStorageDriver(dataverseAlias, superuserApiToken); originalStorageDriver.prettyPrint(); originalStorageDriver.then().assertThat() - .body("data.message", equalTo("undefined")) + .body("data.name", equalTo("undefined")) .statusCode(200); Response setStorageDriverToS3 = UtilIT.setStorageDriver(dataverseAlias, driverLabel, superuserApiToken); @@ -491,7 +491,7 @@ public void testDirectUploadDetectStataFile() { Response originalStorageDriver = UtilIT.getStorageDriver(dataverseAlias, superuserApiToken); originalStorageDriver.prettyPrint(); originalStorageDriver.then().assertThat() - .body("data.message", equalTo("undefined")) + .body("data.name", equalTo("undefined")) .statusCode(200); Response setStorageDriverToS3 = UtilIT.setStorageDriver(dataverseAlias, driverLabel, superuserApiToken); @@ -689,7 +689,7 @@ public void testDirectUploadWithFileCountLimit() throws JsonParseException { Response originalStorageDriver = UtilIT.getStorageDriver(dataverseAlias, superuserApiToken); originalStorageDriver.prettyPrint(); originalStorageDriver.then().assertThat() - .body("data.message", equalTo("undefined")) + .body("data.name", equalTo("undefined")) .statusCode(200); Response setStorageDriverToS3 = UtilIT.setStorageDriver(dataverseAlias, driverLabel, superuserApiToken); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 24f2adbb3ed..e3e62a7a412 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4505,6 +4505,12 @@ static Response setDatasetStorageDriver(Integer datasetId, String driverLabel, S .put("/api/datasets/" + datasetId + "/storageDriver"); } + static Response getDatasetStorageDriver(Integer datasetId, String apiToken) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/datasets/" + datasetId + "/storageDriver"); + } + /** GET on /api/admin/savedsearches/list */ static Response getSavedSearchList() { return given().get("/api/admin/savedsearches/list"); From 8e8b420aae3a2325096f0d893e70794db74e9ca4 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 5 Aug 2025 14:21:21 -0400 Subject: [PATCH 071/634] remove unused import --- .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index a7bf0d637fd..06995d4943c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -39,7 +39,6 @@ import edu.harvard.iq.dataverse.workflow.Workflow; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepData; -import java.io.IOException; import java.util.*; import jakarta.json.Json; import jakarta.json.JsonArrayBuilder; From 648641c04ac5518289df28107df0e7859e4a1cf0 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 5 Aug 2025 14:23:22 -0400 Subject: [PATCH 072/634] add test --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 2ad726aaecf..0828f3d2789 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -6010,6 +6010,7 @@ public void testSetGetDatasetStorageDriver() { getDriver.then().assertThat() .body("data.name", CoreMatchers.equalTo(name)) .body("data.type", CoreMatchers.notNullValue()) + .body("data.label", CoreMatchers.notNullValue()) .statusCode(OK.getStatusCode()); } From 2c03617bfb5889a7c2b88cffeca68ce6d3bde096 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 7 Aug 2025 09:37:23 -0400 Subject: [PATCH 073/634] #11710 dummy endpoint to start --- .../java/edu/harvard/iq/dataverse/api/Dataverses.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 6b7e783a3eb..1523703893a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1763,6 +1763,15 @@ public Response linkDataverse(@Context ContainerRequestContext crc, @PathParam(" return ex.getResponse(); } } + + @GET + @AuthRequired + @Path("/linkingDataverses/{searchTerm}") + public Response getLinkingDataverseList(){ + return null; + } + + @GET @AuthRequired From d37725bc61606e524e52cce60e26bad9d04b24d6 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 7 Aug 2025 13:20:05 -0400 Subject: [PATCH 074/634] #11710 update path --- src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 1523703893a..dba634e8ba9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1766,8 +1766,9 @@ public Response linkDataverse(@Context ContainerRequestContext crc, @PathParam(" @GET @AuthRequired - @Path("/linkingDataverses/{searchTerm}") - public Response getLinkingDataverseList(){ + @Path("{identifier}/linkingDataverses/{searchTerm}") + public Response getLinkingDataverseList(@PathParam("identifier") String dvIdtf, @PathParam("searchTerm") String searchTerm){ + return null; } From 59db4ff859fca947516827f0d3e2dc7c10af2c7b Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 7 Aug 2025 16:24:38 -0400 Subject: [PATCH 075/634] #11710 add api functionality and tests --- .../iq/dataverse/DataverseServiceBean.java | 23 +++++++++++++++---- .../iq/dataverse/api/AbstractApiBean.java | 2 +- .../dataverse/api/DataverseFeaturedItems.java | 2 +- .../harvard/iq/dataverse/api/Dataverses.java | 20 ++++++++++++---- .../iq/dataverse/api/DataversesIT.java | 22 ++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 12 ++++++++++ 6 files changed, 69 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index c14711060af..23d94338992 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -505,18 +505,29 @@ public List filterByAliasQuery(String filterQuery) { return ret; } - public List filterDataversesForLinking(String query, DataverseRequest req, Dataset dataset) { + public List filterDataversesForLinking(String query, DataverseRequest req, DvObject dvo) { List dataverseList = new ArrayList<>(); List results = filterDataversesByNamePattern(query); - if (results == null || results.size() == 0) { + if (results == null || results.isEmpty()) { return null; } + + Dataset linkedDataset = null; + Dataverse linkedDataverse = null; + List alreadyLinkeddv_ids; + + if ((dvo instanceof Dataset)) { + linkedDataset = (Dataset) dvo; + alreadyLinkeddv_ids = em.createNativeQuery("SELECT linkingdataverse_id FROM datasetlinkingdataverse WHERE dataset_id = " + linkedDataset.getId()).getResultList(); + } else { + linkedDataverse = (Dataverse) dvo; + alreadyLinkeddv_ids = em.createNativeQuery("SELECT linkingdataverse_id FROM dataverselinkingdataverse WHERE dataverse_id = " + linkedDataverse.getId()).getResultList(); + } - List alreadyLinkeddv_ids = em.createNativeQuery("SELECT linkingdataverse_id FROM datasetlinkingdataverse WHERE dataset_id = " + dataset.getId()).getResultList(); - List remove = new ArrayList<>(); + List remove = new ArrayList<>(); if (alreadyLinkeddv_ids != null && !alreadyLinkeddv_ids.isEmpty()) { alreadyLinkeddv_ids.stream().map((testDVId) -> this.find(testDVId)).forEachOrdered((removeIt) -> { @@ -526,9 +537,11 @@ public List filterDataversesForLinking(String query, DataverseRequest for (Dataverse res : results) { if (!remove.contains(res)) { - if (this.permissionService.requestOn(req, res).has(Permission.LinkDataset)) { + if ((linkedDataset != null && this.permissionService.requestOn(req, res).has(Permission.LinkDataset)) + || (linkedDataverse != null && this.permissionService.requestOn(req, res).has(Permission.LinkDataverse))) { dataverseList.add(res); } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 76ef91fbd3a..f99727b16db 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -601,7 +601,7 @@ protected DvObject findDvo(@NotNull final String id) throws WrappedResponse { * @throws WrappedResponse */ @NotNull - protected DvObject findDvoByIdAndFeaturedItemTypeOrDie(@NotNull final String dvIdtf, String type) throws WrappedResponse { + protected DvObject findDvoByIdAndTypeOrDie(@NotNull final String dvIdtf, String type) throws WrappedResponse { try { DataverseFeaturedItem.TYPES dvType = DataverseFeaturedItem.getDvType(type); DvObject dvObject = null; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java b/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java index 30c3146fbfb..00f1aa76e7e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java @@ -63,7 +63,7 @@ public Response updateFeaturedItem(@Context ContainerRequestContext crc, if (dataverseFeaturedItem == null) { throw new WrappedResponse(error(Response.Status.NOT_FOUND, MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), id))); } - DvObject dvObject = (dvObjectIdtf != null) ? findDvoByIdAndFeaturedItemTypeOrDie(dvObjectIdtf, type) : null; + DvObject dvObject = (dvObjectIdtf != null) ? findDvoByIdAndTypeOrDie(dvObjectIdtf, type) : null; UpdatedDataverseFeaturedItemDTO updatedDataverseFeaturedItemDTO = UpdatedDataverseFeaturedItemDTO.fromFormData(content, displayOrder, keepFile, imageFileInputStream, contentDispositionHeader, type, dvObject); return ok(json(execCommand(new UpdateDataverseFeaturedItemCommand(createDataverseRequest(getRequestUser(crc)), dataverseFeaturedItem, updatedDataverseFeaturedItemDTO)))); } catch (WrappedResponse e) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index dba634e8ba9..ae49e34d693 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1767,9 +1767,19 @@ public Response linkDataverse(@Context ContainerRequestContext crc, @PathParam(" @GET @AuthRequired @Path("{identifier}/linkingDataverses/{searchTerm}") - public Response getLinkingDataverseList(@PathParam("identifier") String dvIdtf, @PathParam("searchTerm") String searchTerm){ - - return null; + public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @PathParam("searchTerm") String searchTerm, @FormDataParam("type") String type){ + //first determine what you are linking based on identifier and type + try{ + DvObject dvObject = findDvoByIdAndTypeOrDie(dvIdtf, type); + List dataversesForLinking = dataverseService.filterDataversesForLinking(searchTerm, createDataverseRequest(getRequestUser(crc)), dvObject); + JsonArrayBuilder dvBuilder = Json.createArrayBuilder(); + for (Dataverse dv : dataversesForLinking) { + dvBuilder.add(dv.getAlias()); + } + return ok(dvBuilder); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } } @@ -1813,7 +1823,7 @@ public Response createFeaturedItem(@Context ContainerRequestContext crc, try { dataverse = findDataverseOrDie(dvIdtf); if (dvObjectIdtf != null) { - dvObject = findDvoByIdAndFeaturedItemTypeOrDie(dvObjectIdtf, type); + dvObject = findDvoByIdAndTypeOrDie(dvObjectIdtf, type); } } catch (WrappedResponse wr) { return wr.getResponse(); @@ -1901,7 +1911,7 @@ public Response updateFeaturedItems( // ignore dvObject if the id is missing or an empty string DvObject dvObject = dvObjectIdtf.get(i) != null && !dvObjectIdtf.get(i).isEmpty() - ? findDvoByIdAndFeaturedItemTypeOrDie(dvObjectIdtf.get(i), types.get(i)) : null; + ? findDvoByIdAndTypeOrDie(dvObjectIdtf.get(i), types.get(i)) : null; if (ids.get(i) == 0) { newItems.add(NewDataverseFeaturedItemDTO.fromFormData( contents.get(i), displayOrders.get(i), fileInputStream, contentDisposition, types.get(i), dvObject)); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 3e1a160c9f2..4d9f8aa9509 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -677,6 +677,28 @@ public void testImportDDI() throws IOException, InterruptedException { Response deleteUserResponse = UtilIT.deleteUser(username); assertEquals(200, deleteUserResponse.getStatusCode()); } + + @Test + public void testGetLinkableDataverses(){ + Response createUser = UtilIT.createRandomUser(); + String username = UtilIT.getUsernameFromResponse(createUser); + Response makeSuperUser = UtilIT.makeSuperUser(username); + assertEquals(200, makeSuperUser.getStatusCode()); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); + assertEquals(200, publishDataverse.getStatusCode()); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + String datasetPersistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse); + + Response getLinkableDataverses = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiToken, dataverseAlias); + getLinkableDataverses.prettyPrint(); + } @Test public void testImport() throws IOException, InterruptedException { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 24f2adbb3ed..77b4303393e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4773,6 +4773,18 @@ static Response updateDataverseFeaturedItem(long featuredItemId, String apiToken) { return updateDataverseFeaturedItem(featuredItemId, content, displayOrder, keepFile, pathToFile, null, null, apiToken); } + + static Response getLinkableDataverses (String type, String dvObjectId, String apiToken, String dataverseAlias) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .contentType("application/json") + .get("/api/dataverses/" + dataverseAlias + "/featuredItems"); + + /* + {identifier}/linkingDataverses/{searchTerm} + */ + + } static Response updateDataverseFeaturedItem(long featuredItemId, String content, int displayOrder, From 1203c435472d980fc648d3b25940f8d2d6cad5a2 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 8 Aug 2025 16:54:52 -0400 Subject: [PATCH 076/634] #11710 fix test --- .../harvard/iq/dataverse/api/Dataverses.java | 14 +++++++++-- .../iq/dataverse/api/DataversesIT.java | 9 +++++++- .../edu/harvard/iq/dataverse/api/UtilIT.java | 23 +++++++++++-------- 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index ae49e34d693..21b67d1c3c9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1766,9 +1766,16 @@ public Response linkDataverse(@Context ContainerRequestContext crc, @PathParam(" @GET @AuthRequired - @Path("{identifier}/linkingDataverses/{searchTerm}") - public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @PathParam("searchTerm") String searchTerm, @FormDataParam("type") String type){ + @Produces(MediaType.APPLICATION_JSON) + @Consumes(MediaType.MULTIPART_FORM_DATA) + @Path("{identifier}/{type}/linkingDataverses/{searchTerm}") + public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @PathParam("searchTerm") String searchTerm, @PathParam("type") String type){ //first determine what you are linking based on identifier and type + System.out.print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"); + System.out.print("in dataverses method"); + System.out.print("id: " + dvIdtf); + System.out.print("searchTerm: " + searchTerm); + System.out.print("type: " + type); try{ DvObject dvObject = findDvoByIdAndTypeOrDie(dvIdtf, type); List dataversesForLinking = dataverseService.filterDataversesForLinking(searchTerm, createDataverseRequest(getRequestUser(crc)), dvObject); @@ -1779,6 +1786,9 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P return ok(dvBuilder); } catch (WrappedResponse wr) { return wr.getResponse(); + } catch (Exception e){ + return error(Status.BAD_REQUEST, e.getLocalizedMessage()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 4d9f8aa9509..e13d83f6f88 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -696,8 +696,15 @@ public void testGetLinkableDataverses(){ createDatasetResponse.prettyPrint(); String datasetPersistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse); - Response getLinkableDataverses = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiToken, dataverseAlias); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + UtilIT.publishDatasetViaNativeApi(datasetPersistentId, "major", apiToken); + + System.out.print("After pub dataset"); + + Response getLinkableDataverses = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiToken, "dv0"); getLinkableDataverses.prettyPrint(); + getLinkableDataverses.then().assertThat() + .statusCode(OK.getStatusCode()); } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 77b4303393e..f28c0b3e640 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4774,16 +4774,21 @@ static Response updateDataverseFeaturedItem(long featuredItemId, return updateDataverseFeaturedItem(featuredItemId, content, displayOrder, keepFile, pathToFile, null, null, apiToken); } - static Response getLinkableDataverses (String type, String dvObjectId, String apiToken, String dataverseAlias) { - return given() + static Response getLinkableDataverses (String type, String dvObjectId, String apiToken, String searchTerm) { + + String idInPath = dvObjectId; // Assume it's a number to start. + String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. + if (type.equals("dataset")) { + if (!NumberUtils.isCreatable(idInPath)) { + idInPath = ":persistentId"; + optionalQueryParam = "?persistentId=" + dvObjectId; + } + } + + return given() .header(API_TOKEN_HTTP_HEADER, apiToken) - .contentType("application/json") - .get("/api/dataverses/" + dataverseAlias + "/featuredItems"); - - /* - {identifier}/linkingDataverses/{searchTerm} - */ - + .get("/api/dataverses/" + idInPath + "/" + type + "/linkingDataverses/" + searchTerm + optionalQueryParam); + } static Response updateDataverseFeaturedItem(long featuredItemId, String content, From 8e8b084c038a0b103314e203e5b735ea624330f3 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 11 Aug 2025 08:49:06 -0400 Subject: [PATCH 077/634] #11710 add "mini" dv to json printer --- .../java/edu/harvard/iq/dataverse/api/Dataverses.java | 2 +- .../harvard/iq/dataverse/util/json/JsonPrinter.java | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 21b67d1c3c9..a6aa24f417b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1781,7 +1781,7 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P List dataversesForLinking = dataverseService.filterDataversesForLinking(searchTerm, createDataverseRequest(getRequestUser(crc)), dvObject); JsonArrayBuilder dvBuilder = Json.createArrayBuilder(); for (Dataverse dv : dataversesForLinking) { - dvBuilder.add(dv.getAlias()); + dvBuilder.add(json(dv, true)); } return ok(dvBuilder); } catch (WrappedResponse wr) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 592a893083c..ec9abdfa7cc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -269,6 +269,17 @@ public static JsonObjectBuilder json(Workflow wf){ return bld; } + + public static JsonObjectBuilder json(Dataverse dv, Boolean mini) { + if (!mini){ + return json(dv, false, false, null); + } else { + return jsonObjectBuilder() + .add("id", dv.getId()) + .add("alias", dv.getAlias()) + .add("name", dv.getName()); + } + } public static JsonObjectBuilder json(Dataverse dv) { return json(dv, false, false, null); From bb40ca4b40e35a9f2315606913f2c309af363213 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 11 Aug 2025 14:42:05 -0400 Subject: [PATCH 078/634] #11710 fix templates test for existing databases/templates --- .../iq/dataverse/api/DataversesIT.java | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index e13d83f6f88..53643e3d5a5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -2314,17 +2314,39 @@ public void testUpdateInputLevelDisplayOnCreateOverride() { } @Test - public void testCreateAndGetTemplates() { + public void testCreateAndGetTemplates() throws JsonParseException { Response createUserResponse = UtilIT.createRandomUser(); String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); Response createSecondUserResponse = UtilIT.createRandomUser(); String secondApiToken = UtilIT.getApiTokenFromResponse(createSecondUserResponse); - + /* + We need to make this a non-inherited metadatablocks so the get template will only get templates from current dv + */ + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + String newName = "New Test Dataverse Name"; + String newAffiliation = "New Test Dataverse Affiliation"; + String newDataverseType = Dataverse.DataverseType.TEACHING_COURSES.toString(); + String[] newContactEmails = new String[]{"new_email@dataverse.com"}; + String[] newInputLevelNames = new String[]{"geographicCoverage"}; + String[] newFacetIds = new String[]{"contributorName"}; + String[] newMetadataBlockNames = new String[]{"citation", "geospatial", "biomedical"}; + + // Assert that the error is returned for having both MetadataBlockNames and inheritMetadataBlocksFromParent + Response updateDataverseResponse = UtilIT.updateDataverse( + dataverseAlias, dataverseAlias, newName, newAffiliation, newDataverseType, newContactEmails, newInputLevelNames, + null, newMetadataBlockNames, apiToken, + Boolean.FALSE, Boolean.FALSE, null + ); + updateDataverseResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + + // Create a template String jsonString = """ From 791edfb2a4237eb28ae2ef71f9dd76b3e23d4a33 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 12 Aug 2025 09:47:49 -0400 Subject: [PATCH 079/634] #11710 code cleanup --- src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 53643e3d5a5..889f9589819 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -2320,6 +2320,7 @@ public void testCreateAndGetTemplates() throws JsonParseException { Response createSecondUserResponse = UtilIT.createRandomUser(); String secondApiToken = UtilIT.getApiTokenFromResponse(createSecondUserResponse); + /* We need to make this a non-inherited metadatablocks so the get template will only get templates from current dv */ @@ -2342,10 +2343,9 @@ public void testCreateAndGetTemplates() throws JsonParseException { null, newMetadataBlockNames, apiToken, Boolean.FALSE, Boolean.FALSE, null ); + updateDataverseResponse.then().assertThat() .statusCode(OK.getStatusCode()); - - // Create a template @@ -2384,6 +2384,7 @@ public void testCreateAndGetTemplates() throws JsonParseException { jsonString, apiToken ); + createTemplateResponse.then().assertThat().statusCode(OK.getStatusCode()) .body("data.name", equalTo("Dataverse template")) .body("data.usageCount", equalTo(0)) From 61dae90a8f203b6dca0df9c5317e70c40df40d24 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 12 Aug 2025 10:51:39 -0400 Subject: [PATCH 080/634] #11710 add test for perms --- .../harvard/iq/dataverse/api/Dataverses.java | 6 +-- .../iq/dataverse/api/DataversesIT.java | 46 ++++++++++++++++--- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index a6aa24f417b..f4788120ae5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1771,11 +1771,7 @@ public Response linkDataverse(@Context ContainerRequestContext crc, @PathParam(" @Path("{identifier}/{type}/linkingDataverses/{searchTerm}") public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @PathParam("searchTerm") String searchTerm, @PathParam("type") String type){ //first determine what you are linking based on identifier and type - System.out.print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"); - System.out.print("in dataverses method"); - System.out.print("id: " + dvIdtf); - System.out.print("searchTerm: " + searchTerm); - System.out.print("type: " + type); + try{ DvObject dvObject = findDvoByIdAndTypeOrDie(dvIdtf, type); List dataversesForLinking = dataverseService.filterDataversesForLinking(searchTerm, createDataverseRequest(getRequestUser(crc)), dvObject); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 889f9589819..d35270de1a3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -681,30 +681,62 @@ public void testImportDDI() throws IOException, InterruptedException { @Test public void testGetLinkableDataverses(){ Response createUser = UtilIT.createRandomUser(); - String username = UtilIT.getUsernameFromResponse(createUser); - Response makeSuperUser = UtilIT.makeSuperUser(username); - assertEquals(200, makeSuperUser.getStatusCode()); String apiToken = UtilIT.getApiTokenFromResponse(createUser); Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + //Create dataverse for linking + Response createDataverseResponseForLinking = UtilIT.createRandomDataverse(apiToken); + String dataverseAliasForLinking = UtilIT.getAliasFromResponse(createDataverseResponseForLinking); Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); assertEquals(200, publishDataverse.getStatusCode()); + publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAliasForLinking, apiToken); + assertEquals(200, publishDataverse.getStatusCode()); + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); createDatasetResponse.prettyPrint(); String datasetPersistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse); Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); UtilIT.publishDatasetViaNativeApi(datasetPersistentId, "major", apiToken); - - System.out.print("After pub dataset"); - Response getLinkableDataverses = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiToken, "dv0"); + Response getLinkableDataverses = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiToken, dataverseAliasForLinking); getLinkableDataverses.prettyPrint(); getLinkableDataverses.then().assertThat() - .statusCode(OK.getStatusCode()); + .statusCode(OK.getStatusCode()) + .body("data[0].alias", equalTo(dataverseAliasForLinking)); + + Response getLinkableDataversesForDataverse = UtilIT.getLinkableDataverses("dataverse", dataverseAlias, apiToken, dataverseAliasForLinking); + getLinkableDataversesForDataverse.prettyPrint(); + getLinkableDataverses.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].alias", equalTo(dataverseAliasForLinking)); + + // create new user and dataverse - the new dataverse should not be available to the first user for linking... + Response createUserTwo = UtilIT.createRandomUser(); + String apiTokenTwo = UtilIT.getApiTokenFromResponse(createUserTwo); + + //Create dataverse that should be unavailable for linking + Response createDataverseResponseUnavailableForLinking = UtilIT.createRandomDataverse(apiTokenTwo); + createDataverseResponseUnavailableForLinking.prettyPrint(); + String dataverseAliasUnavailableForLinking = UtilIT.getAliasFromResponse(createDataverseResponseUnavailableForLinking); + + Response getUnavailableForDataset = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiToken, dataverseAliasUnavailableForLinking); + getUnavailableForDataset.prettyPrint(); + getUnavailableForDataset.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", equalTo(0)); + + Response getUnavailableForDataverse = UtilIT.getLinkableDataverses("dataverse", dataverseAlias, apiToken, dataverseAliasUnavailableForLinking); + getUnavailableForDataverse.prettyPrint(); + getUnavailableForDataverse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", equalTo(0)); + + } @Test From 32d06ed75bbb7f06a25d856a37963978855cc8a2 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 12 Aug 2025 11:02:07 -0400 Subject: [PATCH 081/634] #11710 add publish to test --- src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index d35270de1a3..678616d7db8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -723,6 +723,8 @@ public void testGetLinkableDataverses(){ Response createDataverseResponseUnavailableForLinking = UtilIT.createRandomDataverse(apiTokenTwo); createDataverseResponseUnavailableForLinking.prettyPrint(); String dataverseAliasUnavailableForLinking = UtilIT.getAliasFromResponse(createDataverseResponseUnavailableForLinking); + publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAliasUnavailableForLinking, apiTokenTwo); + assertEquals(200, publishDataverse.getStatusCode()); Response getUnavailableForDataset = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiToken, dataverseAliasUnavailableForLinking); getUnavailableForDataset.prettyPrint(); From 59572c9485e49f2d00d1857034819c07b7337a47 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 12 Aug 2025 14:20:51 -0400 Subject: [PATCH 082/634] #11710 code cleanup --- src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java | 1 - src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 23d94338992..b8ce8cb1a6b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -541,7 +541,6 @@ public List filterDataversesForLinking(String query, DataverseRequest || (linkedDataverse != null && this.permissionService.requestOn(req, res).has(Permission.LinkDataverse))) { dataverseList.add(res); } - } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index f4788120ae5..8bfa3594602 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1767,7 +1767,6 @@ public Response linkDataverse(@Context ContainerRequestContext crc, @PathParam(" @GET @AuthRequired @Produces(MediaType.APPLICATION_JSON) - @Consumes(MediaType.MULTIPART_FORM_DATA) @Path("{identifier}/{type}/linkingDataverses/{searchTerm}") public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @PathParam("searchTerm") String searchTerm, @PathParam("type") String type){ //first determine what you are linking based on identifier and type From d6625a4692ecfbf06ab026379e35afedaa4e1deb Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 12 Aug 2025 14:53:07 -0400 Subject: [PATCH 083/634] #11710 test that linked dv's are removed from list --- .../iq/dataverse/api/DataversesIT.java | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 678616d7db8..5358debacda 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -682,7 +682,8 @@ public void testImportDDI() throws IOException, InterruptedException { public void testGetLinkableDataverses(){ Response createUser = UtilIT.createRandomUser(); String apiToken = UtilIT.getApiTokenFromResponse(createUser); - + String username = UtilIT.getUsernameFromResponse(createUser); + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); @@ -737,7 +738,25 @@ public void testGetLinkableDataverses(){ getUnavailableForDataverse.then().assertThat() .statusCode(OK.getStatusCode()) .body("data.size()", equalTo(0)); - + + //now link a dataverse and see that it's unavailable in the future + + Response makeSuperUser = UtilIT.setSuperuserStatus(username, Boolean.TRUE); + + Response linkDataset = UtilIT.linkDataset(datasetPersistentId, dataverseAliasForLinking, apiToken); + linkDataset.prettyPrint(); + linkDataset.then().assertThat() + .statusCode(OK.getStatusCode()); + + //set it back to non-super user so perms are limited + UtilIT.setSuperuserStatus(username, Boolean.FALSE); + + //should get an empty list because dataset is already linked + getLinkableDataverses = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiToken, dataverseAliasForLinking); + getLinkableDataverses.prettyPrint(); + getLinkableDataverses.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", equalTo(0)); } From 12f69ffaae42873c9008238e4fd9dfc7c013fd5a Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 12 Aug 2025 15:00:04 -0400 Subject: [PATCH 084/634] #11710 clean up comments --- src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 5358debacda..7782dea8fea 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -739,7 +739,7 @@ public void testGetLinkableDataverses(){ .statusCode(OK.getStatusCode()) .body("data.size()", equalTo(0)); - //now link a dataverse and see that it's unavailable in the future + //now link a dataset and see that it's unavailable in the future Response makeSuperUser = UtilIT.setSuperuserStatus(username, Boolean.TRUE); @@ -748,7 +748,7 @@ public void testGetLinkableDataverses(){ linkDataset.then().assertThat() .statusCode(OK.getStatusCode()); - //set it back to non-super user so perms are limited + //set user api back to non-super user so perms are limited UtilIT.setSuperuserStatus(username, Boolean.FALSE); //should get an empty list because dataset is already linked From 8456b90a1cc015ff3f935217bdf79649f43f93c6 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 12 Aug 2025 15:30:32 -0400 Subject: [PATCH 085/634] Update native-api.rst --- doc/sphinx-guides/source/api/native-api.rst | 44 +++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 8de396e14b3..53270476437 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -732,6 +732,50 @@ Note: you must have "Add Dataset" permission in the given collection to invoke t .. _featured-collections: +List Dataverse Collections that a given Dataset or Dataverse Collection may be linked to +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The user may provide a search term to limit the list of Dataverse Collections returned +The response is a JSON array of the ids, aliases, and names of the Dataverse collections that a given Dataset or Dataverse Collection may be linked to: + +For a given Dataverse Collection: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export OBJECT_TYPE=dataverse + export ID=collectionAlias + export SEARCH_TERM=searchOn + + curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/dataverses/$ID/$OBJECT_TYPE/linkingDataverses/$SEARCH_TERM" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/dataverses/collectionAlias/dataverse/linkingDataverses/searchOn" + +For a given Dataset: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export OBJECT_TYPE=dataset + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB + export SEARCH_TERM=searchOn + + curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/dataverses/:persistentId/$OBJECT_TYPE/linkingDataverses/$SEARCH_TERM?persistentId=$PERSISTENT_IDENTIFIER"" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/dataverses/:persistentId/dataset/linkingDataverses/searchOn?persistentId=doi:10.5072/FK2/J8SJZB" + + + List Featured Collections for a Dataverse Collection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From bb74b24000878104063f0ddf816290388c8aafcc Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 12 Aug 2025 15:32:50 -0400 Subject: [PATCH 086/634] Update native-api.rst --- doc/sphinx-guides/source/api/native-api.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 53270476437..9a861c353d7 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -732,11 +732,11 @@ Note: you must have "Add Dataset" permission in the given collection to invoke t .. _featured-collections: -List Dataverse Collections that a given Dataset or Dataverse Collection may be linked to -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +List Dataverse Collections to which a given Dataset or Dataverse Collection may be linked +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The user may provide a search term to limit the list of Dataverse Collections returned -The response is a JSON array of the ids, aliases, and names of the Dataverse collections that a given Dataset or Dataverse Collection may be linked to: +The user may provide a search term to limit the list of Dataverse Collections returned. +The response is a JSON array of the ids, aliases, and names of the Dataverse collections to which a given Dataset or Dataverse Collection may be linked: For a given Dataverse Collection: From 01de521e93b58c5dd7cc89cc6edf5795a8d60434 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 12 Aug 2025 16:53:21 -0400 Subject: [PATCH 087/634] #11710 add test for partial search term --- .../edu/harvard/iq/dataverse/api/Dataverses.java | 12 +++++++----- .../edu/harvard/iq/dataverse/api/DataversesIT.java | 12 +++++++++++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 8bfa3594602..4d11cc0152a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1771,14 +1771,16 @@ public Response linkDataverse(@Context ContainerRequestContext crc, @PathParam(" public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @PathParam("searchTerm") String searchTerm, @PathParam("type") String type){ //first determine what you are linking based on identifier and type - try{ - DvObject dvObject = findDvoByIdAndTypeOrDie(dvIdtf, type); - List dataversesForLinking = dataverseService.filterDataversesForLinking(searchTerm, createDataverseRequest(getRequestUser(crc)), dvObject); - JsonArrayBuilder dvBuilder = Json.createArrayBuilder(); + try { + DvObject dvObject = findDvoByIdAndTypeOrDie(dvIdtf, type); + List dataversesForLinking = dataverseService.filterDataversesForLinking(searchTerm, createDataverseRequest(getRequestUser(crc)), dvObject); + JsonArrayBuilder dvBuilder = Json.createArrayBuilder(); + if (dataversesForLinking != null && !dataversesForLinking.isEmpty()) { for (Dataverse dv : dataversesForLinking) { dvBuilder.add(json(dv, true)); } - return ok(dvBuilder); + } + return ok(dvBuilder); } catch (WrappedResponse wr) { return wr.getResponse(); } catch (Exception e){ diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 7782dea8fea..3967666d798 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -715,7 +715,17 @@ public void testGetLinkableDataverses(){ getLinkableDataverses.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].alias", equalTo(dataverseAliasForLinking)); - + + //Should be able to get based on a partial alias... + // Partial must include the first part of the name + String searchTerm = dataverseAliasForLinking.substring(0, 5); + + Response getLinkableDataversesForDataversePartial = UtilIT.getLinkableDataverses("dataverse", dataverseAlias, apiToken, searchTerm); + getLinkableDataversesForDataversePartial.prettyPrint(); + getLinkableDataversesForDataversePartial.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].alias", equalTo(dataverseAliasForLinking)); + // create new user and dataverse - the new dataverse should not be available to the first user for linking... Response createUserTwo = UtilIT.createRandomUser(); String apiTokenTwo = UtilIT.getApiTokenFromResponse(createUserTwo); From 9e391921948730c5d34663d2bd0bde74b34828df Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 12 Aug 2025 16:56:34 -0400 Subject: [PATCH 088/634] Update native-api.rst --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 9a861c353d7..22a77cf0d4a 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -735,7 +735,7 @@ Note: you must have "Add Dataset" permission in the given collection to invoke t List Dataverse Collections to which a given Dataset or Dataverse Collection may be linked ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The user may provide a search term to limit the list of Dataverse Collections returned. +The user must provide a search term to limit the list of Dataverse Collections returned. The search term will be compared to the name of the Dataverse Collections and must include the beginning of the Dataverse Collections' names. The response is a JSON array of the ids, aliases, and names of the Dataverse collections to which a given Dataset or Dataverse Collection may be linked: For a given Dataverse Collection: From 3157b2461f789f059329f58c1bdc142573fab551 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 13 Aug 2025 10:32:35 -0400 Subject: [PATCH 089/634] Create 11710-get-available-dataverses-api.md --- doc/release-notes/11710-get-available-dataverses-api.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/release-notes/11710-get-available-dataverses-api.md diff --git a/doc/release-notes/11710-get-available-dataverses-api.md b/doc/release-notes/11710-get-available-dataverses-api.md new file mode 100644 index 00000000000..6658d1d8fd5 --- /dev/null +++ b/doc/release-notes/11710-get-available-dataverses-api.md @@ -0,0 +1,5 @@ +### New API endpoint for retrieving a list of Dataverse Collections to which a given Dataset or Dataverse Collection may be linked + +-The end point also takes in a search term which currently must be the start of the collections' names. +-The user calling this API must have Link Dataset or Link Dataverse permission on the Dataverse Collections returned. +-If the Collection has already been linked to the given Dataset or Collection, it will not be returned. From 250e0e4c46107f0de3d106579260786993402614 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 13 Aug 2025 10:45:31 -0400 Subject: [PATCH 090/634] #11710 add cleanup after tests --- .../iq/dataverse/api/DataversesIT.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 3967666d798..34a5c6a6cec 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -729,6 +729,7 @@ public void testGetLinkableDataverses(){ // create new user and dataverse - the new dataverse should not be available to the first user for linking... Response createUserTwo = UtilIT.createRandomUser(); String apiTokenTwo = UtilIT.getApiTokenFromResponse(createUserTwo); + String usernameTwo = UtilIT.getUsernameFromResponse(createUserTwo); //Create dataverse that should be unavailable for linking Response createDataverseResponseUnavailableForLinking = UtilIT.createRandomDataverse(apiTokenTwo); @@ -767,6 +768,29 @@ public void testGetLinkableDataverses(){ getLinkableDataverses.then().assertThat() .statusCode(OK.getStatusCode()) .body("data.size()", equalTo(0)); + + //set user api back to super user for cleanup + UtilIT.setSuperuserStatus(username, Boolean.TRUE); + + + // Clean up + Response destroyDatasetResponse = UtilIT.destroyDataset(datasetId, apiToken); + assertEquals(200, destroyDatasetResponse.getStatusCode()); + + Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); + assertEquals(200, deleteDataverseResponse.getStatusCode()); + + deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAliasForLinking, apiToken); + assertEquals(200, deleteDataverseResponse.getStatusCode()); + + deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAliasUnavailableForLinking, apiToken); + assertEquals(200, deleteDataverseResponse.getStatusCode()); + + Response deleteUserResponse = UtilIT.deleteUser(usernameTwo); + assertEquals(200, deleteUserResponse.getStatusCode()); + + deleteUserResponse = UtilIT.deleteUser(username); + assertEquals(200, deleteUserResponse.getStatusCode()); } From 9be5fb38f77b150f6a7d22be46cba109ac8e089f Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 13 Aug 2025 14:21:17 -0400 Subject: [PATCH 091/634] Update native-api.rst --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 22a77cf0d4a..2cdcbc65eb4 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -766,7 +766,7 @@ For a given Dataset: export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB export SEARCH_TERM=searchOn - curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/dataverses/:persistentId/$OBJECT_TYPE/linkingDataverses/$SEARCH_TERM?persistentId=$PERSISTENT_IDENTIFIER"" + curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/dataverses/:persistentId/$OBJECT_TYPE/linkingDataverses/$SEARCH_TERM?persistentId=$PERSISTENT_IDENTIFIER" The fully expanded example above (without environment variables) looks like this: From d6404b9142d51b74a5e29fdb8aedbe6d1e1133e2 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 14 Aug 2025 16:55:47 -0400 Subject: [PATCH 092/634] #11387 update command to allow for leaving input levels alone --- .../impl/AbstractWriteDataverseCommand.java | 8 ++- .../iq/dataverse/api/DataversesIT.java | 53 ++++++++++++++++--- .../edu/harvard/iq/dataverse/api/UtilIT.java | 20 +++++++ 3 files changed, 73 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java index 8227572da3b..e4fd5373c7d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java @@ -105,10 +105,14 @@ private void processInputLevels(CommandContext ctxt) { ctxt.fieldTypeInputLevels().deleteDataverseFieldTypeInputLevelFor(dataverse); } else { dataverse.addInputLevelsMetadataBlocksIfNotPresent(inputLevels); - ctxt.fieldTypeInputLevels().deleteDataverseFieldTypeInputLevelFor(dataverse); + //if levels not empty either create or update (handled by save - update when id not null create if null) inputLevels.forEach(inputLevel -> { + DataverseFieldTypeInputLevel ftil = ctxt.fieldTypeInputLevels().findByDataverseIdDatasetFieldTypeId(dataverse.getId(), inputLevel.getDatasetFieldType().getId()); + if(ftil != null){ + inputLevel.setId(ftil.getId()); + } inputLevel.setDataverse(dataverse); - ctxt.fieldTypeInputLevels().create(inputLevel); + ctxt.fieldTypeInputLevels().save(inputLevel); }); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 3e1a160c9f2..50f171de352 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1255,6 +1255,45 @@ public void testUpdateInputLevels() { updateDataverseInputLevelsResponse.then().assertThat() .body("message", equalTo("Error while updating dataverse input levels: Input level list cannot be null or empty")) .statusCode(INTERNAL_SERVER_ERROR.getStatusCode()); + + //Add new types and see that previously changed ones remain as before... #11387 + testInputLevelNames = new String[]{"subtitle", "relatedMaterial"}; + + testRequiredInputLevels = new boolean[] {false, false}; + testIncludedInputLevels = new boolean[] {true, true}; + boolean [] testDisplayOnCreate = new boolean[] {true, true}; + updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, testRequiredInputLevels, testIncludedInputLevels, testDisplayOnCreate, apiToken); + actualInputLevelName = updateDataverseInputLevelsResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); + int subtitleInputLevelIndex = actualInputLevelName.equals("subtitle") ? 0 : 1; + updateDataverseInputLevelsResponse.prettyPrint(); + + updateDataverseInputLevelsResponse.then().assertThat() + .body(String.format("data.inputLevels[%d].include", subtitleInputLevelIndex), equalTo(true)) + .body(String.format("data.inputLevels[%d].required", subtitleInputLevelIndex), equalTo(false)) + .body(String.format("data.inputLevels[%d].displayOnCreate", subtitleInputLevelIndex), equalTo(true)) + .body(String.format("data.inputLevels[%d].include", 1 - subtitleInputLevelIndex), equalTo(true)) + .body(String.format("data.inputLevels[%d].required", 1 - subtitleInputLevelIndex), equalTo(false)) + .body(String.format("data.inputLevels[%d].displayOnCreate", 1 - subtitleInputLevelIndex), equalTo(true)) + .statusCode(OK.getStatusCode()); + actualFieldTypeName1 = updateDataverseInputLevelsResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); + actualFieldTypeName2 = updateDataverseInputLevelsResponse.then().extract().path("data.inputLevels[1].datasetFieldTypeName"); + assertNotEquals(actualFieldTypeName1, actualFieldTypeName2); + assertThat(testInputLevelNames, hasItemInArray(actualFieldTypeName1)); + assertThat(testInputLevelNames, hasItemInArray(actualFieldTypeName2)); + + + testInputLevelNames = new String[]{"subtitle", "otherReferences"}; + testRequiredInputLevels = new boolean[] {false, false}; + testIncludedInputLevels = new boolean[] {true, true}; + testDisplayOnCreate = new boolean[] {false, true}; + + updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, testRequiredInputLevels, testIncludedInputLevels, testDisplayOnCreate, apiToken); + updateDataverseInputLevelsResponse.prettyPrint(); + + updateDataverseInputLevelsResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + } @Test @@ -2274,14 +2313,16 @@ public void testUpdateInputLevelDisplayOnCreateOverride() { .body("data[0].displayName", equalTo("Citation Metadata")) .body("data.size()", equalTo(expectedOnlyDisplayedOnCreateNumberOfMetadataBlocks)) .body("data[0].fields.author.childFields.size()", is(4)); - + updateResponse = UtilIT.updateDataverseInputLevelDisplayOnCreate( - dataverseAlias, "subtitle", false, apiToken); + dataverseAlias, "subtitle", false, apiToken); + String actualInputLevelName = updateResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); + int subtitleIndex = actualInputLevelName.equals("subtitle") ? 0 : 1; updateResponse.then().assertThat() - .statusCode(OK.getStatusCode()) - .body("data.inputLevels[0].displayOnCreate", equalTo(false)) - .body("data.inputLevels[0].datasetFieldTypeName", equalTo("subtitle")); - + .body(String.format("data.inputLevels[%d].displayOnCreate", subtitleIndex), equalTo(false)) + .body(String.format("data.inputLevels[%d].datasetFieldTypeName", subtitleIndex), equalTo("subtitle")) + .statusCode(OK.getStatusCode()); + } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 24f2adbb3ed..8c858a17ec9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4578,6 +4578,18 @@ public static Response updateDataverseInputLevels(String dataverseAlias, String[ .contentType(ContentType.JSON) .put("/api/dataverses/" + dataverseAlias + "/inputLevels"); } + + public static Response updateDataverseInputLevels(String dataverseAlias, String[] inputLevelNames, boolean[] requiredInputLevels, boolean[] includedInputLevels, boolean[] displayOnCreate, String apiToken) { + JsonArrayBuilder inputLevelsArrayBuilder = Json.createArrayBuilder(); + for (int i = 0; i < inputLevelNames.length; i++) { + inputLevelsArrayBuilder.add(createInputLevelObject(inputLevelNames[i], requiredInputLevels[i], includedInputLevels[i], displayOnCreate[i])); + } + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .body(inputLevelsArrayBuilder.build().toString()) + .contentType(ContentType.JSON) + .put("/api/dataverses/" + dataverseAlias + "/inputLevels"); + } private static JsonObjectBuilder createInputLevelObject(String name, boolean required, boolean include) { return Json.createObjectBuilder() @@ -4585,6 +4597,14 @@ private static JsonObjectBuilder createInputLevelObject(String name, boolean req .add("required", required) .add("include", include); } + + private static JsonObjectBuilder createInputLevelObject(String name, boolean required, boolean include, boolean displayOnCreate) { + return Json.createObjectBuilder() + .add("datasetFieldTypeName", name) + .add("required", required) + .add("include", include) + .add("displayOnCreate", displayOnCreate); + } public static Response getOpenAPI(String accept, String format) { Response response = given() From 2788a6d7e5d2959fef406c1dee47dc16081a7ab1 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 15 Aug 2025 10:42:46 -0400 Subject: [PATCH 093/634] #11387 update test; cleanup --- .../edu/harvard/iq/dataverse/api/DataversesIT.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 50f171de352..b07c3e4ffa0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1289,10 +1289,21 @@ public void testUpdateInputLevels() { updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, testRequiredInputLevels, testIncludedInputLevels, testDisplayOnCreate, apiToken); updateDataverseInputLevelsResponse.prettyPrint(); + + actualInputLevelName = updateDataverseInputLevelsResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); + subtitleInputLevelIndex = actualInputLevelName.equals("subtitle") ? 0 : 1; + //make sure subtitle got changed to false updateDataverseInputLevelsResponse.then().assertThat() + .body(String.format("data.inputLevels[%d].displayOnCreate", subtitleInputLevelIndex), equalTo(false)) .statusCode(OK.getStatusCode()); + //make superuser for cleanup + String username = UtilIT.getUsernameFromResponse(createUserResponse); + UtilIT.setSuperuserStatus(username, Boolean.TRUE); + Response deleteDataverse1Response = UtilIT.deleteDataverse(dataverseAlias, apiToken); + deleteDataverse1Response.prettyPrint(); + assertEquals(200, deleteDataverse1Response.getStatusCode()); } From 08dfd98b0bbc4165068bf077c952865db526770d Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 15 Aug 2025 11:31:23 -0400 Subject: [PATCH 094/634] #11387 fix test --- .../engine/command/impl/CreateDataverseCommandTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommandTest.java index 73880b78e7b..2d2fd943d4d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommandTest.java @@ -307,7 +307,7 @@ public void testCustomOptions() throws CommandException { i++; } - assertTrue( dftilsDeleted ); + // assertTrue( dftilsDeleted ); we no longer delete when adding new input levels to preserve previously created for ( DataverseFieldTypeInputLevel dftil : createdDftils ) { assertEquals( result, dftil.getDataverse() ); } From 949b61758c7712cc91ab998c9f427e303b810bd9 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 15 Aug 2025 15:43:07 -0400 Subject: [PATCH 095/634] #11387 source format --- src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index b07c3e4ffa0..36901548f2d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -2324,9 +2324,8 @@ public void testUpdateInputLevelDisplayOnCreateOverride() { .body("data[0].displayName", equalTo("Citation Metadata")) .body("data.size()", equalTo(expectedOnlyDisplayedOnCreateNumberOfMetadataBlocks)) .body("data[0].fields.author.childFields.size()", is(4)); - - updateResponse = UtilIT.updateDataverseInputLevelDisplayOnCreate( - dataverseAlias, "subtitle", false, apiToken); + + updateResponse = UtilIT.updateDataverseInputLevelDisplayOnCreate(dataverseAlias, "subtitle", false, apiToken); String actualInputLevelName = updateResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); int subtitleIndex = actualInputLevelName.equals("subtitle") ? 0 : 1; updateResponse.then().assertThat() From a451f375b7bf9b497d997d1e188aef19b6ac6683 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 18 Aug 2025 15:32:50 -0400 Subject: [PATCH 096/634] #11710 change search term to query param --- src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 4 ++-- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 22f12ca06ff..b771275a112 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1775,8 +1775,8 @@ public Response linkDataverse(@Context ContainerRequestContext crc, @PathParam(" @GET @AuthRequired @Produces(MediaType.APPLICATION_JSON) - @Path("{identifier}/{type}/linkingDataverses/{searchTerm}") - public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @PathParam("searchTerm") String searchTerm, @PathParam("type") String type){ + @Path("{identifier}/{type}/linkingDataverses") + public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @QueryParam("searchTerm") String searchTerm, @PathParam("type") String type){ //first determine what you are linking based on identifier and type try { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index f28c0b3e640..fc2adc16682 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4781,13 +4781,13 @@ static Response getLinkableDataverses (String type, String dvObjectId, String ap if (type.equals("dataset")) { if (!NumberUtils.isCreatable(idInPath)) { idInPath = ":persistentId"; - optionalQueryParam = "?persistentId=" + dvObjectId; + optionalQueryParam = "&persistentId=" + dvObjectId; } } return given() .header(API_TOKEN_HTTP_HEADER, apiToken) - .get("/api/dataverses/" + idInPath + "/" + type + "/linkingDataverses/" + searchTerm + optionalQueryParam); + .get("/api/dataverses/" + idInPath + "/" + type + "/linkingDataverses?searchTerm=" + searchTerm + optionalQueryParam); } static Response updateDataverseFeaturedItem(long featuredItemId, From 645dd38079c39632b0c07d6b5f8721fa0a2d63ad Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 18 Aug 2025 16:12:04 -0400 Subject: [PATCH 097/634] #11710 test for empty search --- .../edu/harvard/iq/dataverse/DataverseServiceBean.java | 4 ++++ .../java/edu/harvard/iq/dataverse/api/DataversesIT.java | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index b8ce8cb1a6b..6e3a3dcc1c8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -535,6 +535,10 @@ public List filterDataversesForLinking(String query, DataverseRequest }); } + if (dvo instanceof Dataverse dataverse){ + remove.add(dataverse); + } + for (Dataverse res : results) { if (!remove.contains(res)) { if ((linkedDataset != null && this.permissionService.requestOn(req, res).has(Permission.LinkDataset)) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 34a5c6a6cec..167aa4ad756 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -721,6 +721,15 @@ public void testGetLinkableDataverses(){ String searchTerm = dataverseAliasForLinking.substring(0, 5); Response getLinkableDataversesForDataversePartial = UtilIT.getLinkableDataverses("dataverse", dataverseAlias, apiToken, searchTerm); + getLinkableDataversesForDataversePartial.prettyPrint(); + getLinkableDataversesForDataversePartial.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].alias", equalTo(dataverseAliasForLinking)); + + + //Try with empty string search term + searchTerm = ""; + getLinkableDataversesForDataversePartial = UtilIT.getLinkableDataverses("dataverse", dataverseAlias, apiToken, searchTerm); getLinkableDataversesForDataversePartial.prettyPrint(); getLinkableDataversesForDataversePartial.then().assertThat() .statusCode(OK.getStatusCode()) From 694cd3431cf4fc5c92e405551d873df1242dd8bb Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 19 Aug 2025 10:13:03 -0400 Subject: [PATCH 098/634] Update native-api.rst --- doc/sphinx-guides/source/api/native-api.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 2e0d7a6f230..0a208967f07 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -735,7 +735,7 @@ Note: you must have "Add Dataset" permission in the given collection to invoke t List Dataverse Collections to which a given Dataset or Dataverse Collection may be linked ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The user must provide a search term to limit the list of Dataverse Collections returned. The search term will be compared to the name of the Dataverse Collections and must include the beginning of the Dataverse Collections' names. +The user may provide a search term to limit the list of Dataverse Collections returned. The search term will be compared to the name of the Dataverse Collections and must include the beginning of the Dataverse Collections' names. The response is a JSON array of the ids, aliases, and names of the Dataverse collections to which a given Dataset or Dataverse Collection may be linked: For a given Dataverse Collection: @@ -748,13 +748,13 @@ For a given Dataverse Collection: export ID=collectionAlias export SEARCH_TERM=searchOn - curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/dataverses/$ID/$OBJECT_TYPE/linkingDataverses/$SEARCH_TERM" + curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/dataverses/$ID/$OBJECT_TYPE/linkingDataverses?searchTerm=$SEARCH_TERM" The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/dataverses/collectionAlias/dataverse/linkingDataverses/searchOn" + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/dataverses/collectionAlias/dataverse/linkingDataverses?searchTerm=searchOn" For a given Dataset: @@ -766,13 +766,13 @@ For a given Dataset: export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB export SEARCH_TERM=searchOn - curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/dataverses/:persistentId/$OBJECT_TYPE/linkingDataverses/$SEARCH_TERM?persistentId=$PERSISTENT_IDENTIFIER" + curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/dataverses/:persistentId/$OBJECT_TYPE/linkingDataverses?searchTerm=SEARCH_TERM&persistentId=$PERSISTENT_IDENTIFIER" The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/dataverses/:persistentId/dataset/linkingDataverses/searchOn?persistentId=doi:10.5072/FK2/J8SJZB" + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/dataverses/:persistentId/dataset/linkingDataverses?searchTerm=searchOn&persistentId=doi:10.5072/FK2/J8SJZB" From 2644fc73dbeeb7abf8acebb601781403f4e7eb9c Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 19 Aug 2025 15:38:02 -0400 Subject: [PATCH 099/634] #11387 get actual index for test --- .../java/edu/harvard/iq/dataverse/api/DataversesIT.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 36901548f2d..e851134a8c7 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -2306,12 +2306,15 @@ public void testUpdateInputLevelDisplayOnCreateOverride() { .body("data.inputLevels[0].displayOnCreate", equalTo(true)) .body("data.inputLevels[0].datasetFieldTypeName", equalTo("notesText")); + String actualInputLevelName = updateResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); + int subtitleInputLevelIndex = actualInputLevelName.equals("subtitle") ? 0 : 1; + updateResponse = UtilIT.updateDataverseInputLevelDisplayOnCreate( dataverseAlias, "subtitle", true, apiToken); updateResponse.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data.inputLevels[0].displayOnCreate", equalTo(true)) - .body("data.inputLevels[0].datasetFieldTypeName", equalTo("subtitle")); + .body(String.format("data.inputLevels[%d].displayOnCreate", subtitleInputLevelIndex), equalTo(true)) + .body(String.format("data.inputLevels[%d].datasetFieldTypeName", subtitleInputLevelIndex), equalTo("subtitle")); listMetadataBlocksResponse = UtilIT.listMetadataBlocks(dataverseAlias, true, true, apiToken); listMetadataBlocksResponse.prettyPrint(); @@ -2326,7 +2329,7 @@ public void testUpdateInputLevelDisplayOnCreateOverride() { .body("data[0].fields.author.childFields.size()", is(4)); updateResponse = UtilIT.updateDataverseInputLevelDisplayOnCreate(dataverseAlias, "subtitle", false, apiToken); - String actualInputLevelName = updateResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); + actualInputLevelName = updateResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); int subtitleIndex = actualInputLevelName.equals("subtitle") ? 0 : 1; updateResponse.then().assertThat() .body(String.format("data.inputLevels[%d].displayOnCreate", subtitleIndex), equalTo(false)) From a9f7f078b76bd2681b4fcb79ac6f62fed8984b26 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 19 Aug 2025 17:04:16 -0400 Subject: [PATCH 100/634] #11387 fix get index for test --- .../java/edu/harvard/iq/dataverse/api/DataversesIT.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index e851134a8c7..17256d3317b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -2306,11 +2306,13 @@ public void testUpdateInputLevelDisplayOnCreateOverride() { .body("data.inputLevels[0].displayOnCreate", equalTo(true)) .body("data.inputLevels[0].datasetFieldTypeName", equalTo("notesText")); - String actualInputLevelName = updateResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); - int subtitleInputLevelIndex = actualInputLevelName.equals("subtitle") ? 0 : 1; - + updateResponse = UtilIT.updateDataverseInputLevelDisplayOnCreate( dataverseAlias, "subtitle", true, apiToken); + + String actualInputLevelName = updateResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); + int subtitleInputLevelIndex = actualInputLevelName.equals("subtitle") ? 0 : 1; + updateResponse.prettyPrint(); updateResponse.then().assertThat() .statusCode(OK.getStatusCode()) .body(String.format("data.inputLevels[%d].displayOnCreate", subtitleInputLevelIndex), equalTo(true)) From 04f6534e4da7a3ff373fe9e72fb1aba371fbf97c Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 20 Aug 2025 11:20:58 -0400 Subject: [PATCH 101/634] #11387 guard against differing json array in test --- .../iq/dataverse/api/DataversesIT.java | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 17256d3317b..e650cd11f6f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1261,19 +1261,31 @@ public void testUpdateInputLevels() { testRequiredInputLevels = new boolean[] {false, false}; testIncludedInputLevels = new boolean[] {true, true}; - boolean [] testDisplayOnCreate = new boolean[] {true, true}; + boolean [] testDisplayOnCreate = new boolean[] {true, false}; updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, testRequiredInputLevels, testIncludedInputLevels, testDisplayOnCreate, apiToken); - actualInputLevelName = updateDataverseInputLevelsResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); - int subtitleInputLevelIndex = actualInputLevelName.equals("subtitle") ? 0 : 1; - updateDataverseInputLevelsResponse.prettyPrint(); - + updateDataverseInputLevelsResponse.prettyPrint(); + int subtitleInputLevelIndex = 0; + int relatedMaterialInputLevelIndex = 0; + int i = 0; + + while (updateDataverseInputLevelsResponse.then().extract().path(String.format("data.inputLevels[%d].datasetFieldTypeName", i)) != null){ + actualInputLevelName = updateDataverseInputLevelsResponse.then().extract().path(String.format("data.inputLevels[%d].datasetFieldTypeName", i)).toString(); + if (actualInputLevelName.equals("subtitle")){ + subtitleInputLevelIndex = i; + } + if (actualInputLevelName.equals("relatedMaterial")){ + relatedMaterialInputLevelIndex = i; + } + i++; + } + updateDataverseInputLevelsResponse.then().assertThat() .body(String.format("data.inputLevels[%d].include", subtitleInputLevelIndex), equalTo(true)) .body(String.format("data.inputLevels[%d].required", subtitleInputLevelIndex), equalTo(false)) .body(String.format("data.inputLevels[%d].displayOnCreate", subtitleInputLevelIndex), equalTo(true)) - .body(String.format("data.inputLevels[%d].include", 1 - subtitleInputLevelIndex), equalTo(true)) - .body(String.format("data.inputLevels[%d].required", 1 - subtitleInputLevelIndex), equalTo(false)) - .body(String.format("data.inputLevels[%d].displayOnCreate", 1 - subtitleInputLevelIndex), equalTo(true)) + .body(String.format("data.inputLevels[%d].include", relatedMaterialInputLevelIndex), equalTo(true)) + .body(String.format("data.inputLevels[%d].required", relatedMaterialInputLevelIndex), equalTo(false)) + .body(String.format("data.inputLevels[%d].displayOnCreate", relatedMaterialInputLevelIndex), equalTo(false)) .statusCode(OK.getStatusCode()); actualFieldTypeName1 = updateDataverseInputLevelsResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); actualFieldTypeName2 = updateDataverseInputLevelsResponse.then().extract().path("data.inputLevels[1].datasetFieldTypeName"); From e001ff048dcb7593acbc2b7b44a46aef8eded424 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 20 Aug 2025 13:26:27 -0400 Subject: [PATCH 102/634] #11387 more array testing fun --- .../iq/dataverse/api/DataversesIT.java | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index e650cd11f6f..682b2ddc8a3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1287,8 +1287,9 @@ public void testUpdateInputLevels() { .body(String.format("data.inputLevels[%d].required", relatedMaterialInputLevelIndex), equalTo(false)) .body(String.format("data.inputLevels[%d].displayOnCreate", relatedMaterialInputLevelIndex), equalTo(false)) .statusCode(OK.getStatusCode()); - actualFieldTypeName1 = updateDataverseInputLevelsResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); - actualFieldTypeName2 = updateDataverseInputLevelsResponse.then().extract().path("data.inputLevels[1].datasetFieldTypeName"); + + actualFieldTypeName1 = updateDataverseInputLevelsResponse.then().extract().path(String.format("data.inputLevels[%d].datasetFieldTypeName", subtitleInputLevelIndex)); + actualFieldTypeName2 = updateDataverseInputLevelsResponse.then().extract().path(String.format("data.inputLevels[%d].datasetFieldTypeName", relatedMaterialInputLevelIndex)); assertNotEquals(actualFieldTypeName1, actualFieldTypeName2); assertThat(testInputLevelNames, hasItemInArray(actualFieldTypeName1)); assertThat(testInputLevelNames, hasItemInArray(actualFieldTypeName2)); @@ -1302,9 +1303,21 @@ public void testUpdateInputLevels() { updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, testRequiredInputLevels, testIncludedInputLevels, testDisplayOnCreate, apiToken); updateDataverseInputLevelsResponse.prettyPrint(); - actualInputLevelName = updateDataverseInputLevelsResponse.then().extract().path("data.inputLevels[0].datasetFieldTypeName"); - subtitleInputLevelIndex = actualInputLevelName.equals("subtitle") ? 0 : 1; - + subtitleInputLevelIndex = 0; + int otherReferencesInputLevelIndex = 0; + i = 0; + + while (updateDataverseInputLevelsResponse.then().extract().path(String.format("data.inputLevels[%d].datasetFieldTypeName", i)) != null) { + actualInputLevelName = updateDataverseInputLevelsResponse.then().extract().path(String.format("data.inputLevels[%d].datasetFieldTypeName", i)).toString(); + if (actualInputLevelName.equals("subtitle")) { + subtitleInputLevelIndex = i; + } + if (actualInputLevelName.equals("otherReferences")) { + otherReferencesInputLevelIndex = i; + } + i++; + } + //make sure subtitle got changed to false updateDataverseInputLevelsResponse.then().assertThat() .body(String.format("data.inputLevels[%d].displayOnCreate", subtitleInputLevelIndex), equalTo(false)) From de9f68768555affdad2ef0e2d193e362246c6de4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Aug 2025 00:11:29 +0000 Subject: [PATCH 103/634] Bump actions/setup-java from 4 to 5 Bumps [actions/setup-java](https://github.com/actions/setup-java) from 4 to 5. - [Release notes](https://github.com/actions/setup-java/releases) - [Commits](https://github.com/actions/setup-java/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-java dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/container_app_pr.yml | 2 +- .github/workflows/deploy_beta_testing.yml | 2 +- .github/workflows/maven_cache_management.yml | 2 +- .github/workflows/maven_unit_test.yml | 6 +++--- .github/workflows/spi_release.yml | 6 +++--- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/container_app_pr.yml b/.github/workflows/container_app_pr.yml index a4c52805156..b9f9fc6a51c 100644 --- a/.github/workflows/container_app_pr.yml +++ b/.github/workflows/container_app_pr.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/checkout@v4 with: ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge' - - uses: actions/setup-java@v4 + - uses: actions/setup-java@v5 with: java-version: "17" distribution: 'adopt' diff --git a/.github/workflows/deploy_beta_testing.yml b/.github/workflows/deploy_beta_testing.yml index 7a236a316fb..dc49c51e9d3 100644 --- a/.github/workflows/deploy_beta_testing.yml +++ b/.github/workflows/deploy_beta_testing.yml @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/setup-java@v5 with: distribution: 'zulu' java-version: '17' diff --git a/.github/workflows/maven_cache_management.yml b/.github/workflows/maven_cache_management.yml index fedf63b7c54..8099119760d 100644 --- a/.github/workflows/maven_cache_management.yml +++ b/.github/workflows/maven_cache_management.yml @@ -36,7 +36,7 @@ jobs: - name: Determine Java version from Parent POM run: echo "JAVA_VERSION=$(grep '' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} - name: Set up JDK ${{ env.JAVA_VERSION }} - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: ${{ env.JAVA_VERSION }} distribution: temurin diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index f0cf461d8e6..5335efc2c76 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -39,7 +39,7 @@ jobs: # Basic setup chores - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: ${{ matrix.jdk }} distribution: temurin @@ -105,7 +105,7 @@ jobs: # Basic setup chores - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: ${{ matrix.jdk }} distribution: temurin @@ -138,7 +138,7 @@ jobs: # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/setup-java@v5 with: java-version: '17' distribution: temurin diff --git a/.github/workflows/spi_release.yml b/.github/workflows/spi_release.yml index 6398edca412..1c896aaff6f 100644 --- a/.github/workflows/spi_release.yml +++ b/.github/workflows/spi_release.yml @@ -38,7 +38,7 @@ jobs: if: github.event_name == 'pull_request' && needs.check-secrets.outputs.available == 'true' steps: - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/setup-java@v5 with: java-version: '17' distribution: 'adopt' @@ -64,7 +64,7 @@ jobs: if: github.event_name == 'push' && needs.check-secrets.outputs.available == 'true' steps: - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/setup-java@v5 with: java-version: '17' distribution: 'adopt' @@ -76,7 +76,7 @@ jobs: # Running setup-java again overwrites the settings.xml - IT'S MANDATORY TO DO THIS SECOND SETUP!!! - name: Set up Maven Central Repository - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: '17' distribution: 'adopt' From 9d24e0946bea91110b6f25c809133ee2aafb14bb Mon Sep 17 00:00:00 2001 From: Florian Fritze Date: Thu, 21 Aug 2025 15:33:33 +0200 Subject: [PATCH 104/634] integer detection for array access --- .../edu/harvard/iq/dataverse/DatasetFieldServiceBean.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index dce7a98fd75..49fdafd2f5e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -764,7 +764,13 @@ Object processPathSegment(int index, String[] pathParts, JsonValue curPath, Stri } } else { - curPath = ((JsonObject) curPath).get(pathParts[index]); + try { + int indexNumber = Integer.parseInt(pathParts[index]); + curPath = ((JsonObject) curPath).get(indexNumber); + } catch (NumberFormatException nfe) { + curPath = ((JsonObject) curPath).get(pathParts[index]); + } + // curPath = ((JsonObject) curPath).get(pathParts[index]); logger.fine("Found next Path object " + curPath.toString()); return processPathSegment(index + 1, pathParts, curPath, termUri); } From 894d3d5b2ea4eba018c55c7c074a2ca527819835 Mon Sep 17 00:00:00 2001 From: Florian Fritze Date: Fri, 22 Aug 2025 07:27:25 +0200 Subject: [PATCH 105/634] first check if it's a number --- .../iq/dataverse/DatasetFieldServiceBean.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index 49fdafd2f5e..35b34c8d3a5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -43,6 +43,7 @@ import jakarta.persistence.criteria.*; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.http.HttpResponse; import org.apache.http.HttpResponseInterceptor; import org.apache.http.client.methods.HttpGet; @@ -764,10 +765,14 @@ Object processPathSegment(int index, String[] pathParts, JsonValue curPath, Stri } } else { - try { - int indexNumber = Integer.parseInt(pathParts[index]); - curPath = ((JsonObject) curPath).get(indexNumber); - } catch (NumberFormatException nfe) { + if (NumberUtils.isCreatable(pathParts[index])) { + try { + int indexNumber = Integer.parseInt(pathParts[index]); + curPath = ((JsonObject) curPath).get(indexNumber); + } catch (NumberFormatException nfe) { + logger.fine("Please provide a valid integer number " + pathParts[index]); + } + } else { curPath = ((JsonObject) curPath).get(pathParts[index]); } // curPath = ((JsonObject) curPath).get(pathParts[index]); From 69e5a349f3cd44f28b86ff0e1dac4ba5eb71f803 Mon Sep 17 00:00:00 2001 From: Florian Fritze Date: Fri, 22 Aug 2025 08:03:34 +0200 Subject: [PATCH 106/634] JsonArray fix --- .../java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index 35b34c8d3a5..244e7f959d8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -768,7 +768,7 @@ Object processPathSegment(int index, String[] pathParts, JsonValue curPath, Stri if (NumberUtils.isCreatable(pathParts[index])) { try { int indexNumber = Integer.parseInt(pathParts[index]); - curPath = ((JsonObject) curPath).get(indexNumber); + curPath = ((JsonArray) curPath).get(indexNumber); } catch (NumberFormatException nfe) { logger.fine("Please provide a valid integer number " + pathParts[index]); } From 5c1ce14f5abae0ce350ca113caa5deb89153794b Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 26 Aug 2025 11:33:42 -0400 Subject: [PATCH 107/634] #11719 combine perms and search filter for query --- .../iq/dataverse/DataverseServiceBean.java | 41 +++++++++++++++++++ .../iq/dataverse/PermissionServiceBean.java | 24 +++++++++-- .../harvard/iq/dataverse/api/Dataverses.java | 18 +++++++- .../iq/dataverse/api/DataversesIT.java | 17 ++++++-- 4 files changed, 91 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 6e3a3dcc1c8..85d4179a6d9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -550,6 +550,47 @@ public List filterDataversesForLinking(String query, DataverseRequest return dataverseList; } + + + public List removeUnlinkableDataverses (List allWithPerms, DvObject dvo){ + List dataverseList = new ArrayList<>(); + Dataset linkedDataset = null; + Dataverse linkedDataverse = null; + List alreadyLinkeddv_ids; + + if ((dvo instanceof Dataset)) { + linkedDataset = (Dataset) dvo; + alreadyLinkeddv_ids = em.createNativeQuery("SELECT linkingdataverse_id FROM datasetlinkingdataverse WHERE dataset_id = " + linkedDataset.getId()).getResultList(); + } else { + linkedDataverse = (Dataverse) dvo; + alreadyLinkeddv_ids = em.createNativeQuery("SELECT linkingdataverse_id FROM dataverselinkingdataverse WHERE dataverse_id = " + linkedDataverse.getId()).getResultList(); + } + + List remove = new ArrayList<>(); + + if (alreadyLinkeddv_ids != null && !alreadyLinkeddv_ids.isEmpty()) { + alreadyLinkeddv_ids.stream().map((testDVId) -> this.find(testDVId)).forEachOrdered((removeIt) -> { + remove.add(removeIt); + }); + } + + if (dvo instanceof Dataverse dataverse){ + remove.add(dataverse); + } else { + //dataset is always owned by a dataverse + remove.add((Dataverse)dvo.getOwner()); + } + + for (Dataverse res : allWithPerms) { + if (!remove.contains(res)) { + dataverseList.add(res); + } + } + + return dataverseList; + } + + public List filterDataversesForUnLinking(String query, DataverseRequest req, Dataset dataset) { List alreadyLinkeddv_ids = em.createNativeQuery("SELECT linkingdataverse_id FROM datasetlinkingdataverse WHERE dataset_id = " + dataset.getId()).getResultList(); List dataverseList = new ArrayList<>(); diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index f1099c0a439..778bf23586a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -101,7 +101,7 @@ WITH grouplist AS ( WHERE explicitgroup_authenticateduser.containedauthenticatedusers_id = @USERID ) - SELECT * FROM DATAVERSE WHERE id IN ( + SELECT * FROM DATAVERSE WHERE id IN ( SELECT definitionpoint_id FROM roleassignment WHERE roleassignment.assigneeidentifier IN ( @@ -157,7 +157,7 @@ AND EXISTS (SELECT id FROM dataverserole WHERE dataverserole.id = roleassignment AND @IPRANGESQL ) ) - ) + ) @SEARCHCLAUSE """; /** * A request-level permission query (e.g includes IP ras). @@ -921,8 +921,16 @@ private boolean hasUnrestrictedReleasedFiles(DatasetVersion targetDatasetVersion public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, Permission permission) { return findPermittedCollections(request, user, 1 << permission.ordinal()); } + + public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, Permission permission, String searchTerm) { + return findPermittedCollections(request, user, 1 << permission.ordinal(), searchTerm); + } public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit) { + return findPermittedCollections(request, user, permissionBit, ""); + } + + public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit, String searchTerm) { if (user != null) { // IP Group - Only check IP if a User is calling for themself String ipRangeSQL = "FALSE"; @@ -950,16 +958,26 @@ public List findPermittedCollections(DataverseRequest request, Authen } } } + + String searchClause = ""; + if (!searchTerm.isEmpty()){ + searchClause = " AND ((LOWER(DATAVERSE.alias) LIKE '%@ALIAS%') OR (LOWER(DATAVERSE.name) LIKE '%@NAME%') OR (LOWER(DATAVERSE.affiliation) LIKE '%@AFFILIATION%')) " + .replace("@ALIAS", searchTerm.toLowerCase()) + .replace("@NAME", searchTerm.toLowerCase()) + .replace("@AFFILIATION", searchTerm.toLowerCase()); + } String sqlCode = LIST_ALL_DATAVERSES_USER_HAS_PERMISSION .replace("@USERID", String.valueOf(user.getId())) .replace("@PERMISSIONBIT", String.valueOf(permissionBit)) - .replace("@IPRANGESQL", ipRangeSQL); + .replace("@IPRANGESQL", ipRangeSQL) + .replace("@SEARCHCLAUSE", searchClause); return em.createNativeQuery(sqlCode, Dataverse.class).getResultList(); } return null; } + /** * Calculates the complete list of role assignments for a given user on a DvObject. * This includes roles assigned directly to the user and roles inherited from any groups diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 5522b35906b..bf332a5bb3d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -15,6 +15,7 @@ import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroup; import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroupProvider; import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroupServiceBean; +import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.dataset.DatasetType; @@ -1781,8 +1782,21 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P try { DvObject dvObject = findDvoByIdAndTypeOrDie(dvIdtf, type); - List dataversesForLinking = dataverseService.filterDataversesForLinking(searchTerm, createDataverseRequest(getRequestUser(crc)), dvObject); - JsonArrayBuilder dvBuilder = Json.createArrayBuilder(); + // List dataversesForLinking = dataverseService.filterDataversesForLinking(searchTerm, createDataverseRequest(getRequestUser(crc)), dvObject); + // public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, Permission permission, String searchTerm) { + + AuthenticatedUser requestUser = (AuthenticatedUser)getRequestUser(crc); + List dataversesForLinking = new ArrayList<>(); + + if ((dvObject instanceof Dataset)) { + dataversesForLinking = permissionService.findPermittedCollections( new DataverseRequest(requestUser, (IpAddress) null), requestUser, Permission.LinkDataset, searchTerm); + } else { + dataversesForLinking = permissionService.findPermittedCollections( new DataverseRequest(requestUser, (IpAddress) null), requestUser, Permission.LinkDataverse, searchTerm); + + } + + dataversesForLinking = dataverseService.removeUnlinkableDataverses(dataversesForLinking, dvObject); + JsonArrayBuilder dvBuilder = Json.createArrayBuilder(); if (dataversesForLinking != null && !dataversesForLinking.isEmpty()) { for (Dataverse dv : dataversesForLinking) { dvBuilder.add(json(dv, true)); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 167aa4ad756..9ba53f2e8b6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -683,7 +683,7 @@ public void testGetLinkableDataverses(){ Response createUser = UtilIT.createRandomUser(); String apiToken = UtilIT.getApiTokenFromResponse(createUser); String username = UtilIT.getUsernameFromResponse(createUser); - + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); @@ -718,7 +718,7 @@ public void testGetLinkableDataverses(){ //Should be able to get based on a partial alias... // Partial must include the first part of the name - String searchTerm = dataverseAliasForLinking.substring(0, 5); + String searchTerm = dataverseAliasForLinking.substring(0, 7); Response getLinkableDataversesForDataversePartial = UtilIT.getLinkableDataverses("dataverse", dataverseAlias, apiToken, searchTerm); getLinkableDataversesForDataversePartial.prettyPrint(); @@ -745,15 +745,21 @@ public void testGetLinkableDataverses(){ createDataverseResponseUnavailableForLinking.prettyPrint(); String dataverseAliasUnavailableForLinking = UtilIT.getAliasFromResponse(createDataverseResponseUnavailableForLinking); publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAliasUnavailableForLinking, apiTokenTwo); + publishDataverse.prettyPrint(); assertEquals(200, publishDataverse.getStatusCode()); - Response getUnavailableForDataset = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiToken, dataverseAliasUnavailableForLinking); + //user 3 will not have permissions + Response createUserThree = UtilIT.createRandomUser(); + String apiTokenThree = UtilIT.getApiTokenFromResponse(createUserThree); + String usernameThree = UtilIT.getUsernameFromResponse(createUserThree); + + Response getUnavailableForDataset = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiTokenThree, dataverseAliasUnavailableForLinking); getUnavailableForDataset.prettyPrint(); getUnavailableForDataset.then().assertThat() .statusCode(OK.getStatusCode()) .body("data.size()", equalTo(0)); - Response getUnavailableForDataverse = UtilIT.getLinkableDataverses("dataverse", dataverseAlias, apiToken, dataverseAliasUnavailableForLinking); + Response getUnavailableForDataverse = UtilIT.getLinkableDataverses("dataverse", dataverseAlias, apiTokenThree, dataverseAliasUnavailableForLinking); getUnavailableForDataverse.prettyPrint(); getUnavailableForDataverse.then().assertThat() .statusCode(OK.getStatusCode()) @@ -798,6 +804,9 @@ public void testGetLinkableDataverses(){ Response deleteUserResponse = UtilIT.deleteUser(usernameTwo); assertEquals(200, deleteUserResponse.getStatusCode()); + deleteUserResponse = UtilIT.deleteUser(usernameThree); + assertEquals(200, deleteUserResponse.getStatusCode()); + deleteUserResponse = UtilIT.deleteUser(username); assertEquals(200, deleteUserResponse.getStatusCode()); From 2bcfa709ec04aaca0c5166f3086ebdcedcbd370c Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 26 Aug 2025 14:55:13 -0400 Subject: [PATCH 108/634] #11710 refactor query update --- .../harvard/iq/dataverse/PermissionServiceBean.java | 12 ++---------- .../edu/harvard/iq/dataverse/api/DataversesIT.java | 10 ++++++++-- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index 778bf23586a..6fe5db974eb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -157,7 +157,7 @@ AND EXISTS (SELECT id FROM dataverserole WHERE dataverserole.id = roleassignment AND @IPRANGESQL ) ) - ) @SEARCHCLAUSE + ) AND ((LOWER(DATAVERSE.alias) LIKE '%@SEARCHTERM%') OR (LOWER(DATAVERSE.name) LIKE '%@SEARCHTERM%') OR (LOWER(DATAVERSE.affiliation) LIKE '%@SEARCHTERM%')) """; /** * A request-level permission query (e.g includes IP ras). @@ -959,19 +959,11 @@ public List findPermittedCollections(DataverseRequest request, Authen } } - String searchClause = ""; - if (!searchTerm.isEmpty()){ - searchClause = " AND ((LOWER(DATAVERSE.alias) LIKE '%@ALIAS%') OR (LOWER(DATAVERSE.name) LIKE '%@NAME%') OR (LOWER(DATAVERSE.affiliation) LIKE '%@AFFILIATION%')) " - .replace("@ALIAS", searchTerm.toLowerCase()) - .replace("@NAME", searchTerm.toLowerCase()) - .replace("@AFFILIATION", searchTerm.toLowerCase()); - } - String sqlCode = LIST_ALL_DATAVERSES_USER_HAS_PERMISSION .replace("@USERID", String.valueOf(user.getId())) .replace("@PERMISSIONBIT", String.valueOf(permissionBit)) .replace("@IPRANGESQL", ipRangeSQL) - .replace("@SEARCHCLAUSE", searchClause); + .replace("@SEARCHTERM", searchTerm); return em.createNativeQuery(sqlCode, Dataverse.class).getResultList(); } return null; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 9ba53f2e8b6..64c88197494 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -753,17 +753,23 @@ public void testGetLinkableDataverses(){ String apiTokenThree = UtilIT.getApiTokenFromResponse(createUserThree); String usernameThree = UtilIT.getUsernameFromResponse(createUserThree); - Response getUnavailableForDataset = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiTokenThree, dataverseAliasUnavailableForLinking); + Response getUnavailableForDataset = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiToken, dataverseAliasUnavailableForLinking); getUnavailableForDataset.prettyPrint(); getUnavailableForDataset.then().assertThat() .statusCode(OK.getStatusCode()) .body("data.size()", equalTo(0)); - Response getUnavailableForDataverse = UtilIT.getLinkableDataverses("dataverse", dataverseAlias, apiTokenThree, dataverseAliasUnavailableForLinking); + Response getUnavailableForDataverse = UtilIT.getLinkableDataverses("dataverse", dataverseAlias, apiToken, dataverseAliasUnavailableForLinking); getUnavailableForDataverse.prettyPrint(); getUnavailableForDataverse.then().assertThat() .statusCode(OK.getStatusCode()) .body("data.size()", equalTo(0)); + + Response getNoPermsOnAnyCollection = UtilIT.getLinkableDataverses("dataverse", dataverseAlias, apiTokenThree, ""); + getNoPermsOnAnyCollection.prettyPrint(); + getNoPermsOnAnyCollection.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", equalTo(0)); //now link a dataset and see that it's unavailable in the future From 36f36f749107d6ce28caec381885bf4d3b1ca8f3 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 26 Aug 2025 15:01:14 -0400 Subject: [PATCH 109/634] Expanding the ExportDataProvider interface to add hooks for more efficient handling of the raw data for datasets with massive numbers of ingested/tabular datafiles and variables. #11766 #11405 --- modules/dataverse-spi/pom.xml | 2 +- .../gdcc/spi/export/ExportDataProvider.java | 28 +++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/modules/dataverse-spi/pom.xml b/modules/dataverse-spi/pom.xml index b00053fe5e0..8c31a2a4026 100644 --- a/modules/dataverse-spi/pom.xml +++ b/modules/dataverse-spi/pom.xml @@ -13,7 +13,7 @@ io.gdcc dataverse-spi - 2.0.0${project.version.suffix} + 2.1.0${project.version.suffix} jar Dataverse SPI Plugin API diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java index d039ac39e8f..45b332fb088 100644 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java +++ b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java @@ -21,8 +21,13 @@ public interface ExportDataProvider { * OAI_ORE export are the only two that provide 'complete' * dataset-level metadata along with basic file metadata for each file * in the dataset. + * @param options - optional argument(s). currently supports DatasetMetadataOnly: + * in a situation where we need to generate a format like DC, + * that has no use for file-level metadata, it makes sense to + * skip retrieving and formatting it, since there can be quite a few + * files in a dataset. */ - JsonObject getDatasetJson(); + JsonObject getDatasetJson(ExportDataOption... options); /** * @@ -39,7 +44,7 @@ public interface ExportDataProvider { * Dataverse is capable of extracting DDI-centric metadata from tabular * datafiles. This detailed metadata, which is only available for successfully * "ingested" tabular files, is not included in the output of any other methods - * in this interface. + * in this interface. * * @return - a JSONArray with one entry per ingested tabular dataset file. * @apiNote - there is no JSON schema available for this output and the format @@ -50,6 +55,20 @@ public interface ExportDataProvider { */ JsonArray getDatasetFileDetails(); + /** + * Similar to the above, but + * a) retrieves the information for the ingested/tabular data files _only_ + * b) provides an option for retrieving this stuff in batches + * c) provides an option for skipping restricted/embargoed etc. files. + * Intended for datasets with massive numbers of tabular files and datavariables. + * @param offset (can be null) + * @param length (can be null) + * @param options (optional) supports PublicFilesOnly; + * @return json array containing the datafile/filemetadata->datatable->datavariable metadata + * @throws ExportException + */ + JsonArray getTabularDataDetails(Integer offset, Integer length, ExportDataOption ... options) throws ExportException; + /** * * @return - the subset of metadata conforming to the schema.org standard as @@ -92,5 +111,10 @@ public interface ExportDataProvider { default Optional getPrerequisiteInputStream() { return Optional.empty(); } + + public enum ExportDataOption { + DatasetMetadataOnly, + PublicFilesOnly; + } } From f9963ca3405f158116f198c573418f8ede13932f Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 26 Aug 2025 15:14:34 -0400 Subject: [PATCH 110/634] cosmetic #11766 --- .../src/main/java/io/gdcc/spi/export/ExportDataProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java index 45b332fb088..eefe0d7d828 100644 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java +++ b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java @@ -21,7 +21,7 @@ public interface ExportDataProvider { * OAI_ORE export are the only two that provide 'complete' * dataset-level metadata along with basic file metadata for each file * in the dataset. - * @param options - optional argument(s). currently supports DatasetMetadataOnly: + * @param options - optional argument(s). needs to support ExportDataOption.DatasetMetadataOnly: * in a situation where we need to generate a format like DC, * that has no use for file-level metadata, it makes sense to * skip retrieving and formatting it, since there can be quite a few @@ -63,7 +63,7 @@ public interface ExportDataProvider { * Intended for datasets with massive numbers of tabular files and datavariables. * @param offset (can be null) * @param length (can be null) - * @param options (optional) supports PublicFilesOnly; + * @param options (optional) current use case is ExportDataOption.PublicFilesOnly; * @return json array containing the datafile/filemetadata->datatable->datavariable metadata * @throws ExportException */ From f18f663c347059148571e416973989c835f923c2 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 26 Aug 2025 16:55:59 -0400 Subject: [PATCH 111/634] adding directUpload to the Json response --- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 2 +- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 2 +- .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 5 ++++- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 1 + src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java | 4 ++++ 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 1d726830c31..8354c8e71d2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -2197,7 +2197,7 @@ public Response getStorageDriver(@Context ContainerRequestContext crc, @PathPara return wr.getResponse(); } //Note that this returns what's set directly on this dataverse. If null/DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER, the user would have to recurse the chain of parents to find the effective storageDriver - return ok(JsonPrinter.jsonStorageDriver(dataverse.getStorageDriverId())); + return ok(JsonPrinter.jsonStorageDriver(dataverse.getStorageDriverId(), null)); } @PUT diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index fd7ca0d61cb..fa7f12e0436 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3690,7 +3690,7 @@ public Response getFileStore(@Context ContainerRequestContext crc, @PathParam("i return error(Response.Status.NOT_FOUND, "No such dataset"); } - return ok(JsonPrinter.jsonStorageDriver(dataset.getStorageDriverId())); + return ok(JsonPrinter.jsonStorageDriver(dataset.getStorageDriverId(), dataset)); } @PUT diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 5ac5e4a8fd8..7dfb0ab3c2a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -1634,11 +1634,14 @@ public static JsonArrayBuilder jsonTemplateInstructions(Map temp return jsonArrayBuilder; } - public static JsonObjectBuilder jsonStorageDriver(String storageDriverId) { + public static JsonObjectBuilder jsonStorageDriver(String storageDriverId, Dataset dataset) { JsonObjectBuilder jsonObjectBuilder = new NullSafeJsonBuilder(); jsonObjectBuilder.add("name", storageDriverId); jsonObjectBuilder.add("type", DataAccess.getDriverType(storageDriverId)); jsonObjectBuilder.add("label", DataAccess.getStorageDriverLabelFor(storageDriverId)); + if (dataset != null) { + jsonObjectBuilder.add("directUpload", DataAccess.uploadToDatasetAllowed(dataset, storageDriverId)); + } return jsonObjectBuilder; } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index ecbad14830d..2dd6ff00c7b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -6087,6 +6087,7 @@ public void testSetGetDatasetStorageDriver() { .body("data.name", CoreMatchers.equalTo(name)) .body("data.type", CoreMatchers.notNullValue()) .body("data.label", CoreMatchers.notNullValue()) + .body("data.directUpload", CoreMatchers.notNullValue()) .statusCode(OK.getStatusCode()); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java index d5fecb32937..24625c87ce2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.api; +import org.hamcrest.CoreMatchers; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.ResponseInputStream; @@ -153,6 +154,9 @@ public void testNonDirectUpload() { Response updatedStorageDriver = UtilIT.getStorageDriver(dataverseAlias, superuserApiToken); updatedStorageDriver.prettyPrint(); updatedStorageDriver.then().assertThat() + .body("data.type", CoreMatchers.notNullValue()) + .body("data.label", CoreMatchers.notNullValue()) + .body("data.directUpload", CoreMatchers.nullValue()) .statusCode(200); Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); From 6d1c0900db58af89ca18b1e5e842f6bca29b91df Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 27 Aug 2025 10:20:38 -0400 Subject: [PATCH 112/634] return default storage driver when undefined --- .../edu/harvard/iq/dataverse/api/Admin.java | 2 +- .../harvard/iq/dataverse/api/Datasets.java | 2 +- .../harvard/iq/dataverse/api/DatasetsIT.java | 19 +++++++++++++++++++ .../iq/dataverse/api/DataversesIT.java | 12 ++++++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 8354c8e71d2..71406d9f293 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -2197,7 +2197,7 @@ public Response getStorageDriver(@Context ContainerRequestContext crc, @PathPara return wr.getResponse(); } //Note that this returns what's set directly on this dataverse. If null/DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER, the user would have to recurse the chain of parents to find the effective storageDriver - return ok(JsonPrinter.jsonStorageDriver(dataverse.getStorageDriverId(), null)); + return ok(JsonPrinter.jsonStorageDriver(dataverse.getEffectiveStorageDriverId(), null)); } @PUT diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index fa7f12e0436..151b833d3c4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3690,7 +3690,7 @@ public Response getFileStore(@Context ContainerRequestContext crc, @PathParam("i return error(Response.Status.NOT_FOUND, "No such dataset"); } - return ok(JsonPrinter.jsonStorageDriver(dataset.getStorageDriverId(), dataset)); + return ok(JsonPrinter.jsonStorageDriver(dataset.getEffectiveStorageDriverId(), dataset)); } @PUT diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 2dd6ff00c7b..46da46e68ea 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -6089,6 +6089,25 @@ public void testSetGetDatasetStorageDriver() { .body("data.label", CoreMatchers.notNullValue()) .body("data.directUpload", CoreMatchers.notNullValue()) .statusCode(OK.getStatusCode()); + + // Test dataset under root with default storage driver + Response getStorageDriverResponse = UtilIT.getStorageDriver("root", apiToken); + getStorageDriverResponse.prettyPrint(); + data = JsonUtil.getJsonObject(getStorageDriverResponse.getBody().asString()); + name = data.getJsonObject("data").getString("name"); + String type = data.getJsonObject("data").getString("type"); + String label = data.getJsonObject("data").getString("label"); + createDataset = UtilIT.createRandomDatasetViaNativeApi("root", apiToken); + datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + getDriver = UtilIT.getDatasetStorageDriver(datasetId, apiToken); + getDriver.prettyPrint(); + assertEquals(200, getDriver.getStatusCode()); + getDriver.then().assertThat() + .body("data.name", CoreMatchers.equalTo(name)) + .body("data.type", CoreMatchers.equalTo(type)) + .body("data.label", CoreMatchers.equalTo(label)) + .body("data.directUpload", CoreMatchers.notNullValue()) + .statusCode(OK.getStatusCode()); } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 3e1a160c9f2..64ce24076a0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -2371,6 +2371,18 @@ public void testCreateAndGetTemplates() { getTemplateResponse.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode()); } + @Test + public void testGetStorageDriver() { + Response updatedStorageDriver = UtilIT.getStorageDriver("root", getSuperuserToken()); + updatedStorageDriver.prettyPrint(); + updatedStorageDriver.then().assertThat() + .body("data.name", CoreMatchers.notNullValue()) + .body("data.type", CoreMatchers.notNullValue()) + .body("data.label", CoreMatchers.notNullValue()) + .body("data.directUpload", CoreMatchers.nullValue()) + .statusCode(200); + } + private String getSuperuserToken() { Response createResponse = UtilIT.createRandomUser(); String adminApiToken = UtilIT.getApiTokenFromResponse(createResponse); From b88796c8574ea3b335e50a39ae3b0353b72559f5 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 27 Aug 2025 11:50:13 -0400 Subject: [PATCH 113/634] add new query param to api call and directDownload to json output --- .../11695-change-api-get-storage-driver.md | 2 ++ .../source/admin/dataverses-datasets.rst | 4 ++++ src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 11 ++++++++--- .../harvard/iq/dataverse/util/json/JsonPrinter.java | 6 ++++++ .../java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 4 +++- .../edu/harvard/iq/dataverse/api/DataversesIT.java | 11 ++++++++++- .../java/edu/harvard/iq/dataverse/api/UtilIT.java | 6 +++++- 7 files changed, 38 insertions(+), 6 deletions(-) diff --git a/doc/release-notes/11695-change-api-get-storage-driver.md b/doc/release-notes/11695-change-api-get-storage-driver.md index 3680da4b72c..cd78c642a2a 100644 --- a/doc/release-notes/11695-change-api-get-storage-driver.md +++ b/doc/release-notes/11695-change-api-get-storage-driver.md @@ -7,4 +7,6 @@ The API for getting the Storage Driver info has been changed/extended. /api/admin/dataverse/{dataverse-alias}/storageDriver changed "message" to "name" and added "type" and "label" +Also added query param for /api/admin/dataverse/{dataverse-alias}/storageDriver?defaultToOwner=true to recurse the chain of parents to find the effective storageDriver + See also [the guides](https://dataverse-guide--11664.org.readthedocs.build/en/11664/api/native-api.html#configure-a-dataset-to-store-all-new-files-in-a-specific-file-store), #11695, and #11664. diff --git a/doc/sphinx-guides/source/admin/dataverses-datasets.rst b/doc/sphinx-guides/source/admin/dataverses-datasets.rst index a37819c90e1..fcfbd0c62f3 100644 --- a/doc/sphinx-guides/source/admin/dataverses-datasets.rst +++ b/doc/sphinx-guides/source/admin/dataverses-datasets.rst @@ -60,6 +60,10 @@ The current driver can be seen using:: curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver +Or to recurse the chain of parents to find the effective storageDriver:: + + curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver?defaultToOwner=true + (Note that for ``dataverse.files.store1.label=MyLabel``, ``store1`` will be returned.) and can be reset to the default store with:: diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 71406d9f293..48246820bc7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -2183,7 +2183,8 @@ public Response addRoleAssignementsToChildren(@Context ContainerRequestContext c @GET @AuthRequired @Path("/dataverse/{alias}/storageDriver") - public Response getStorageDriver(@Context ContainerRequestContext crc, @PathParam("alias") String alias) throws WrappedResponse { + public Response getStorageDriver(@Context ContainerRequestContext crc, @PathParam("alias") String alias, + @QueryParam("defaultToOwner") Boolean defaultToOwner) throws WrappedResponse { Dataverse dataverse = dataverseSvc.findByAlias(alias); if (dataverse == null) { return error(Response.Status.NOT_FOUND, "Could not find dataverse based on alias supplied: " + alias + "."); @@ -2196,8 +2197,12 @@ public Response getStorageDriver(@Context ContainerRequestContext crc, @PathPara } catch (WrappedResponse wr) { return wr.getResponse(); } - //Note that this returns what's set directly on this dataverse. If null/DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER, the user would have to recurse the chain of parents to find the effective storageDriver - return ok(JsonPrinter.jsonStorageDriver(dataverse.getEffectiveStorageDriverId(), null)); + + if (defaultToOwner != null && defaultToOwner) { + return ok(JsonPrinter.jsonStorageDriver(dataverse.getEffectiveStorageDriverId(), null)); + } else { + return ok(JsonPrinter.jsonStorageDriver(dataverse.getStorageDriverId(), null)); + } } @PUT diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 7dfb0ab3c2a..a6066b1da64 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -40,6 +40,7 @@ import edu.harvard.iq.dataverse.workflow.Workflow; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepData; +import java.io.IOException; import java.util.*; import jakarta.json.Json; import jakarta.json.JsonArrayBuilder; @@ -1641,6 +1642,11 @@ public static JsonObjectBuilder jsonStorageDriver(String storageDriverId, Datase jsonObjectBuilder.add("label", DataAccess.getStorageDriverLabelFor(storageDriverId)); if (dataset != null) { jsonObjectBuilder.add("directUpload", DataAccess.uploadToDatasetAllowed(dataset, storageDriverId)); + try { + jsonObjectBuilder.add("directDownload", DataAccess.getStorageIO(dataset).downloadRedirectEnabled()); + } catch (IOException ex) { + logger.fine("Failed to get Storage IO for dataset " + ex.getMessage()); + } } return jsonObjectBuilder; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 46da46e68ea..25e4451ac9f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -6088,10 +6088,11 @@ public void testSetGetDatasetStorageDriver() { .body("data.type", CoreMatchers.notNullValue()) .body("data.label", CoreMatchers.notNullValue()) .body("data.directUpload", CoreMatchers.notNullValue()) + .body("data.directDownload", CoreMatchers.notNullValue()) .statusCode(OK.getStatusCode()); // Test dataset under root with default storage driver - Response getStorageDriverResponse = UtilIT.getStorageDriver("root", apiToken); + Response getStorageDriverResponse = UtilIT.getStorageDriver("root", apiToken, Boolean.TRUE); getStorageDriverResponse.prettyPrint(); data = JsonUtil.getJsonObject(getStorageDriverResponse.getBody().asString()); name = data.getJsonObject("data").getString("name"); @@ -6107,6 +6108,7 @@ public void testSetGetDatasetStorageDriver() { .body("data.type", CoreMatchers.equalTo(type)) .body("data.label", CoreMatchers.equalTo(label)) .body("data.directUpload", CoreMatchers.notNullValue()) + .body("data.directDownload", CoreMatchers.notNullValue()) .statusCode(OK.getStatusCode()); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 64ce24076a0..4cc3ddb0be6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.api; +import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.util.json.JsonParseException; import edu.harvard.iq.dataverse.util.json.JsonParser; import edu.harvard.iq.dataverse.util.json.JsonUtil; @@ -2373,13 +2374,21 @@ public void testCreateAndGetTemplates() { @Test public void testGetStorageDriver() { - Response updatedStorageDriver = UtilIT.getStorageDriver("root", getSuperuserToken()); + Response updatedStorageDriver = UtilIT.getStorageDriver("root", getSuperuserToken(), Boolean.TRUE); updatedStorageDriver.prettyPrint(); updatedStorageDriver.then().assertThat() .body("data.name", CoreMatchers.notNullValue()) .body("data.type", CoreMatchers.notNullValue()) .body("data.label", CoreMatchers.notNullValue()) .body("data.directUpload", CoreMatchers.nullValue()) + .body("data.directDownload", CoreMatchers.nullValue()) + .statusCode(200); + + // Root without default is undefined + updatedStorageDriver = UtilIT.getStorageDriver("root", getSuperuserToken(), null); + updatedStorageDriver.prettyPrint(); + updatedStorageDriver.then().assertThat() + .body("data.name", CoreMatchers.equalTo(DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER)) .statusCode(200); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 9cf320666c6..67028d8be06 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2911,9 +2911,13 @@ static Response listStorageDrivers(String apiToken) { } static Response getStorageDriver(String dvAlias, String apiToken) { + return getStorageDriver(dvAlias, apiToken, null); + } + static Response getStorageDriver(String dvAlias, String apiToken, Boolean defaultToOwner) { + String params = defaultToOwner != null ? "?defaultToOwner=" + defaultToOwner : ""; return given() .header(API_TOKEN_HTTP_HEADER, apiToken) - .get("/api/admin/dataverse/" + dvAlias + "/storageDriver"); + .get("/api/admin/dataverse/" + dvAlias + "/storageDriver" + params); } static Response setStorageDriver(String dvAlias, String label, String apiToken) { From 409f77da07a67319c8bf99d4de2373f59ed922bc Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 27 Aug 2025 14:50:30 -0400 Subject: [PATCH 114/634] update Maven snapshot release id and URL #11766 #11512 --- modules/dataverse-spi/pom.xml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/dataverse-spi/pom.xml b/modules/dataverse-spi/pom.xml index 8c31a2a4026..a603e274234 100644 --- a/modules/dataverse-spi/pom.xml +++ b/modules/dataverse-spi/pom.xml @@ -64,11 +64,13 @@ - ossrh - https://s01.oss.sonatype.org/content/repositories/snapshots + central + https://central.sonatype.com/repository/maven-snapshots/ + ossrh + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ @@ -110,7 +112,9 @@ nexus-staging-maven-plugin true + ossrh + https://s01.oss.sonatype.org true From 4ed1879c26e947fcb81d085fa7431541a76c291f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 27 Aug 2025 14:51:08 -0400 Subject: [PATCH 115/634] explain how to publish a maven snapshot locally #11766 #11512 --- .../developers/making-library-releases.rst | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/doc/sphinx-guides/source/developers/making-library-releases.rst b/doc/sphinx-guides/source/developers/making-library-releases.rst index be867f9196a..0daa7fb89db 100755 --- a/doc/sphinx-guides/source/developers/making-library-releases.rst +++ b/doc/sphinx-guides/source/developers/making-library-releases.rst @@ -36,6 +36,32 @@ Releasing a Snapshot Version to Maven Central That is to say, to make a snapshot release, you only need to get one or more commits into the default branch. +It's possible, of course, to make snapshot releases outside of GitHub Actions, from environments such as your laptop. Generally, you'll want to look at the GitHub Action and try to do the equivalent. You'll need a file set up locally at ``~/.m2/settings.xml`` with the following (contact a core developer for the redacted bits): + +.. code-block:: bash + + + + + central + REDACTED + REDACTED + + + + +Then, study the GitHub Action and perform similar commands from your local environment. For example, as of this writing, for the dataverse-spi project, you can run the following commands, substituting the suffix you need: + +``mvn -f modules/dataverse-spi -Dproject.version.suffix="2.1.0-PR11767-SNAPSHOT" verify`` + +``mvn -f modules/dataverse-spi -Dproject.version.suffix="2.1.0-PR11767-SNAPSHOT" deploy`` + +This will upload the snapshot here, for example: https://central.sonatype.com/repository/maven-snapshots/io/gdcc/dataverse-spi/2.1.02.1.0-PR11767-SNAPSHOT/dataverse-spi-2.1.02.1.0-PR11767-20250827.182026-1.jar + +Before OSSRH was retired, you could browse through snapshot jars you published at https://s01.oss.sonatype.org/content/repositories/snapshots/io/gdcc/dataverse-spi/2.0.0-PR9685-SNAPSHOT/, for example. Now, even though you may see the URL of the jar as shown above during the "deploy" step, if you try to browse the various snapshot jars at https://central.sonatype.com/repository/maven-snapshots/io/gdcc/dataverse-spi/2.1.02.1.0-PR11767-SNAPSHOT/ you'll see "This maven2 hosted repository is not directly browseable at this URL. Please use the browse or HTML index views to inspect the contents of this repository." Sadly, the "browse" and "HTML index" links don't work, as noted in a `question `_ on the Sonatype Community forum. Below is a suggestion for confirming that the jar was uploaded properly, which is to use Maven to copy the jar to your local directory. You could then compare checksums. + +``mvn dependency:copy -DrepoUrl=https://central.sonatype.com/repository/maven-snapshots/ -Dartifact=io.gdcc:dataverse-spi:2.1.02.1.0-PR11767-SNAPSHOT -DoutputDirectory=.`` + Releasing a Release (Non-Snapshot) Version to Maven Central ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From c688965eb4a64f4faacc70a164be62e35ee39c41 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 27 Aug 2025 15:00:45 -0400 Subject: [PATCH 116/634] #11710 separate perms test from lookup --- .../iq/dataverse/DataverseServiceBean.java | 51 ++++++++++++++++++ .../iq/dataverse/PermissionServiceBean.java | 14 ++--- .../iq/dataverse/api/AbstractApiBean.java | 6 ++- .../dataverse/api/DataverseFeaturedItems.java | 2 +- .../harvard/iq/dataverse/api/Dataverses.java | 53 ++++++++++--------- 5 files changed, 88 insertions(+), 38 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 85d4179a6d9..d3f96da107e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -687,6 +687,57 @@ public List filterDataversesByNamePattern(String pattern) { return searchResults; } + /* + This method takes a search parameter and expands it into a list of + Dataverses with matching names. + The search is performed on the name with the trailing word "dataverse" + stripped (if present). This way the search on "data" (or on "da" pr + "dat") does NOT return almost every dataverse in the database - since + most of them have names that end in "... Dataverse". + The query isn't pretty, but it works, and it's still EJB QL (and NOT a + native query). + */ + public List filterDataversesByNameAliasPattern(String pattern) { + + pattern = pattern.toLowerCase(); + + String pattern1 = pattern + "%"; + String pattern2 = "% " + pattern + "%"; + + // Adjust the queries for very short, 1 and 2-character patterns: + if (pattern.length() == 1) { + pattern1 = pattern; + pattern2 = pattern + " %"; + } + /*if (pattern.length() == 2) { + pattern2 = pattern + "%"; + }*/ + + + String qstr = "select dv from Dataverse dv " + + "where (LOWER(dv.name) LIKE :dataverse and ((SUBSTRING(LOWER(dv.name),0,(LENGTH(dv.name)-9)) LIKE :pattern1) " + + " or (SUBSTRING(LOWER(dv.name),0,(LENGTH(dv.name)-9)) LIKE :pattern2))) " + + "or (LOWER(dv.name) NOT LIKE :dataverse and ((LOWER(dv.name) LIKE :pattern1) " + + " or (LOWER(dv.name) LIKE :pattern2)) " + + "or (LOWER(dv.alias) LIKE :pattern1) " + + " or (LOWER(dv.alias) LIKE :pattern2))) " + + "order by dv.alias"; + + List searchResults = null; + + try { + searchResults = em.createQuery(qstr, Dataverse.class) + .setParameter("dataverse", "%dataverse") + .setParameter("pattern1", pattern1) + .setParameter("pattern2", pattern2) + .getResultList(); + } catch (Exception ex) { + searchResults = null; + } + + return searchResults; + } + /** * Used to identify and properly display Harvested objects on the dataverse page. * diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index 6fe5db974eb..a4111963ea2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -157,7 +157,7 @@ AND EXISTS (SELECT id FROM dataverserole WHERE dataverserole.id = roleassignment AND @IPRANGESQL ) ) - ) AND ((LOWER(DATAVERSE.alias) LIKE '%@SEARCHTERM%') OR (LOWER(DATAVERSE.name) LIKE '%@SEARCHTERM%') OR (LOWER(DATAVERSE.affiliation) LIKE '%@SEARCHTERM%')) + ) """; /** * A request-level permission query (e.g includes IP ras). @@ -922,15 +922,8 @@ public List findPermittedCollections(DataverseRequest request, Authen return findPermittedCollections(request, user, 1 << permission.ordinal()); } - public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, Permission permission, String searchTerm) { - return findPermittedCollections(request, user, 1 << permission.ordinal(), searchTerm); - } - - public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit) { - return findPermittedCollections(request, user, permissionBit, ""); - } - public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit, String searchTerm) { + public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit) { if (user != null) { // IP Group - Only check IP if a User is calling for themself String ipRangeSQL = "FALSE"; @@ -962,8 +955,7 @@ public List findPermittedCollections(DataverseRequest request, Authen String sqlCode = LIST_ALL_DATAVERSES_USER_HAS_PERMISSION .replace("@USERID", String.valueOf(user.getId())) .replace("@PERMISSIONBIT", String.valueOf(permissionBit)) - .replace("@IPRANGESQL", ipRangeSQL) - .replace("@SEARCHTERM", searchTerm); + .replace("@IPRANGESQL", ipRangeSQL); return em.createNativeQuery(sqlCode, Dataverse.class).getResultList(); } return null; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index f99727b16db..938d7fc9081 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -601,7 +601,7 @@ protected DvObject findDvo(@NotNull final String id) throws WrappedResponse { * @throws WrappedResponse */ @NotNull - protected DvObject findDvoByIdAndTypeOrDie(@NotNull final String dvIdtf, String type) throws WrappedResponse { + protected DvObject findDvoByIdAndTypeOrDie(@NotNull final String dvIdtf, String type, Boolean testForReleased) throws WrappedResponse { try { DataverseFeaturedItem.TYPES dvType = DataverseFeaturedItem.getDvType(type); DvObject dvObject = null; @@ -634,7 +634,9 @@ protected DvObject findDvoByIdAndTypeOrDie(@NotNull final String dvIdtf, String } } } - DataverseFeaturedItem.validateTypeAndDvObject(dvIdtf, dvObject, dvType); + if (testForReleased){ + DataverseFeaturedItem.validateTypeAndDvObject(dvIdtf, dvObject, dvType); + } return dvObject; } catch (IllegalArgumentException e) { throw new WrappedResponse(error(Response.Status.BAD_REQUEST, e.getMessage())); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java b/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java index 00f1aa76e7e..7fbdd79e3c3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java @@ -63,7 +63,7 @@ public Response updateFeaturedItem(@Context ContainerRequestContext crc, if (dataverseFeaturedItem == null) { throw new WrappedResponse(error(Response.Status.NOT_FOUND, MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), id))); } - DvObject dvObject = (dvObjectIdtf != null) ? findDvoByIdAndTypeOrDie(dvObjectIdtf, type) : null; + DvObject dvObject = (dvObjectIdtf != null) ? findDvoByIdAndTypeOrDie(dvObjectIdtf, type, true) : null; UpdatedDataverseFeaturedItemDTO updatedDataverseFeaturedItemDTO = UpdatedDataverseFeaturedItemDTO.fromFormData(content, displayOrder, keepFile, imageFileInputStream, contentDispositionHeader, type, dvObject); return ok(json(execCommand(new UpdateDataverseFeaturedItemCommand(createDataverseRequest(getRequestUser(crc)), dataverseFeaturedItem, updatedDataverseFeaturedItemDTO)))); } catch (WrappedResponse e) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index bf332a5bb3d..f59cfbc3bab 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -70,6 +70,7 @@ import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.StreamingOutput; +import java.sql.PreparedStatement; import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; @@ -120,7 +121,7 @@ public class Dataverses extends AbstractApiBean { @EJB DataverseFeaturedItemServiceBean dataverseFeaturedItemServiceBean; - + @POST @AuthRequired public Response addRoot(@Context ContainerRequestContext crc, String body) { @@ -1777,37 +1778,41 @@ public Response linkDataverse(@Context ContainerRequestContext crc, @PathParam(" @AuthRequired @Produces(MediaType.APPLICATION_JSON) @Path("{identifier}/{type}/linkingDataverses") - public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @QueryParam("searchTerm") String searchTerm, @PathParam("type") String type){ + public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @QueryParam("searchTerm") String searchTerm, @PathParam("type") String type) { //first determine what you are linking based on identifier and type + + AuthenticatedUser requestUser = (AuthenticatedUser) getRequestUser(crc); + DataverseRequest dvReq = new DataverseRequest(requestUser, (IpAddress) null); + List dataversesForLinking; + dataversesForLinking = permissionService.findPermittedCollections(dvReq, requestUser, Permission.LinkDataset); try { - DvObject dvObject = findDvoByIdAndTypeOrDie(dvIdtf, type); - // List dataversesForLinking = dataverseService.filterDataversesForLinking(searchTerm, createDataverseRequest(getRequestUser(crc)), dvObject); - // public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, Permission permission, String searchTerm) { - - AuthenticatedUser requestUser = (AuthenticatedUser)getRequestUser(crc); - List dataversesForLinking = new ArrayList<>(); - - if ((dvObject instanceof Dataset)) { - dataversesForLinking = permissionService.findPermittedCollections( new DataverseRequest(requestUser, (IpAddress) null), requestUser, Permission.LinkDataset, searchTerm); - } else { - dataversesForLinking = permissionService.findPermittedCollections( new DataverseRequest(requestUser, (IpAddress) null), requestUser, Permission.LinkDataverse, searchTerm); - - } - - dataversesForLinking = dataverseService.removeUnlinkableDataverses(dataversesForLinking, dvObject); - JsonArrayBuilder dvBuilder = Json.createArrayBuilder(); - if (dataversesForLinking != null && !dataversesForLinking.isEmpty()) { + + DvObject dvObject = findDvoByIdAndTypeOrDie(dvIdtf, type, false); + List dataversesForLinkingSearch = new ArrayList(); + dataversesForLinkingSearch = dataverseService.filterDataversesByNameAliasPattern(searchTerm); + + List mergedWithSearch = new ArrayList<>(); + dataversesForLinking = dataverseService.removeUnlinkableDataverses(dataversesForLinking, dvObject); + if (!dataversesForLinkingSearch.isEmpty()) { for (Dataverse dv : dataversesForLinking) { + if (dataversesForLinkingSearch.contains(dv)) { + mergedWithSearch.add(dv); + } + } + } + JsonArrayBuilder dvBuilder = Json.createArrayBuilder(); + if (!mergedWithSearch.isEmpty()) { + for (Dataverse dv : mergedWithSearch) { dvBuilder.add(json(dv, true)); } } - return ok(dvBuilder); + return ok(dvBuilder); } catch (WrappedResponse wr) { return wr.getResponse(); - } catch (Exception e){ + } catch (Exception e) { return error(Status.BAD_REQUEST, e.getLocalizedMessage()); - + } } @@ -1852,7 +1857,7 @@ public Response createFeaturedItem(@Context ContainerRequestContext crc, try { dataverse = findDataverseOrDie(dvIdtf); if (dvObjectIdtf != null) { - dvObject = findDvoByIdAndTypeOrDie(dvObjectIdtf, type); + dvObject = findDvoByIdAndTypeOrDie(dvObjectIdtf, type, true); } } catch (WrappedResponse wr) { return wr.getResponse(); @@ -1940,7 +1945,7 @@ public Response updateFeaturedItems( // ignore dvObject if the id is missing or an empty string DvObject dvObject = dvObjectIdtf.get(i) != null && !dvObjectIdtf.get(i).isEmpty() - ? findDvoByIdAndTypeOrDie(dvObjectIdtf.get(i), types.get(i)) : null; + ? findDvoByIdAndTypeOrDie(dvObjectIdtf.get(i), types.get(i), true) : null; if (ids.get(i) == 0) { newItems.add(NewDataverseFeaturedItemDTO.fromFormData( contents.get(i), displayOrders.get(i), fileInputStream, contentDisposition, types.get(i), dvObject)); From f570aaaf411b76f6147d37c06a80fceaaa98962b Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 27 Aug 2025 16:24:05 -0400 Subject: [PATCH 117/634] add TODO comments to spi workflow #11766 #11512 --- .github/workflows/spi_release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/spi_release.yml b/.github/workflows/spi_release.yml index 6398edca412..1d29cdf76c9 100644 --- a/.github/workflows/spi_release.yml +++ b/.github/workflows/spi_release.yml @@ -42,6 +42,7 @@ jobs: with: java-version: '17' distribution: 'adopt' + # TODO: change this from ossrh to central? server-id: ossrh server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD @@ -80,6 +81,7 @@ jobs: with: java-version: '17' distribution: 'adopt' + # TODO: change this from ossrh to central? server-id: ossrh server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD From 96c8b25c2efcaf58ebe47e62fb1356eed56bfdc2 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 27 Aug 2025 16:28:46 -0400 Subject: [PATCH 118/634] #11710 remove unused method --- .../iq/dataverse/DataverseServiceBean.java | 51 ------------------- .../harvard/iq/dataverse/api/Dataverses.java | 12 ++--- 2 files changed, 6 insertions(+), 57 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index d3f96da107e..85d4179a6d9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -687,57 +687,6 @@ public List filterDataversesByNamePattern(String pattern) { return searchResults; } - /* - This method takes a search parameter and expands it into a list of - Dataverses with matching names. - The search is performed on the name with the trailing word "dataverse" - stripped (if present). This way the search on "data" (or on "da" pr - "dat") does NOT return almost every dataverse in the database - since - most of them have names that end in "... Dataverse". - The query isn't pretty, but it works, and it's still EJB QL (and NOT a - native query). - */ - public List filterDataversesByNameAliasPattern(String pattern) { - - pattern = pattern.toLowerCase(); - - String pattern1 = pattern + "%"; - String pattern2 = "% " + pattern + "%"; - - // Adjust the queries for very short, 1 and 2-character patterns: - if (pattern.length() == 1) { - pattern1 = pattern; - pattern2 = pattern + " %"; - } - /*if (pattern.length() == 2) { - pattern2 = pattern + "%"; - }*/ - - - String qstr = "select dv from Dataverse dv " - + "where (LOWER(dv.name) LIKE :dataverse and ((SUBSTRING(LOWER(dv.name),0,(LENGTH(dv.name)-9)) LIKE :pattern1) " - + " or (SUBSTRING(LOWER(dv.name),0,(LENGTH(dv.name)-9)) LIKE :pattern2))) " - + "or (LOWER(dv.name) NOT LIKE :dataverse and ((LOWER(dv.name) LIKE :pattern1) " - + " or (LOWER(dv.name) LIKE :pattern2)) " - + "or (LOWER(dv.alias) LIKE :pattern1) " - + " or (LOWER(dv.alias) LIKE :pattern2))) " - + "order by dv.alias"; - - List searchResults = null; - - try { - searchResults = em.createQuery(qstr, Dataverse.class) - .setParameter("dataverse", "%dataverse") - .setParameter("pattern1", pattern1) - .setParameter("pattern2", pattern2) - .getResultList(); - } catch (Exception ex) { - searchResults = null; - } - - return searchResults; - } - /** * Used to identify and properly display Harvested objects on the dataverse page. * diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index f59cfbc3bab..512b7b00fab 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1780,17 +1780,17 @@ public Response linkDataverse(@Context ContainerRequestContext crc, @PathParam(" @Path("{identifier}/{type}/linkingDataverses") public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @QueryParam("searchTerm") String searchTerm, @PathParam("type") String type) { //first determine what you are linking based on identifier and type - - AuthenticatedUser requestUser = (AuthenticatedUser) getRequestUser(crc); - DataverseRequest dvReq = new DataverseRequest(requestUser, (IpAddress) null); - List dataversesForLinking; - dataversesForLinking = permissionService.findPermittedCollections(dvReq, requestUser, Permission.LinkDataset); try { DvObject dvObject = findDvoByIdAndTypeOrDie(dvIdtf, type, false); List dataversesForLinkingSearch = new ArrayList(); - dataversesForLinkingSearch = dataverseService.filterDataversesByNameAliasPattern(searchTerm); + dataversesForLinkingSearch = dataverseService.filterDataversesByNamePattern(searchTerm); + + AuthenticatedUser requestUser = (AuthenticatedUser) getRequestUser(crc); + DataverseRequest dvReq = new DataverseRequest(requestUser, (IpAddress) null); + List dataversesForLinking; + dataversesForLinking = permissionService.findPermittedCollections(dvReq, requestUser, Permission.LinkDataset); List mergedWithSearch = new ArrayList<>(); dataversesForLinking = dataverseService.removeUnlinkableDataverses(dataversesForLinking, dvObject); From 0500d7e7066dc36c81f640cafcbc63de8682e395 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 27 Aug 2025 17:13:59 -0400 Subject: [PATCH 119/634] Another experimental approach (#11766) --- .../io/gdcc/spi/export/ExportDataOption.java | 50 +++++++++++++++++++ .../gdcc/spi/export/ExportDataProvider.java | 19 +++---- 2 files changed, 57 insertions(+), 12 deletions(-) create mode 100644 modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataOption.java diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataOption.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataOption.java new file mode 100644 index 00000000000..1c447cc53e3 --- /dev/null +++ b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataOption.java @@ -0,0 +1,50 @@ +package io.gdcc.spi.export; + +/** + * + * @author landreev + * Provides a mechanism for defining various data retrieval options for the + * export subsystem in a way that should allow us adding support for more + * options going forward with minimal or no changes to the existing code in + * export plugins. + */ +public class ExportDataOption { + + public enum SupportedOptions { + DatasetMetadataOnly, + PublicFilesOnly; + } + + private SupportedOptions optionType; + + /*public static ExportDataOption addOption(String option) { + ExportDataOption ret = new ExportDataOption(); + + for (SupportedOptions supported : SupportedOptions.values()) { + if (supported.toString().equals(option)) { + ret.optionType = supported; + } + } + return ret; + }*/ + + public static ExportDataOption addDatasetMetadataOnly() { + ExportDataOption ret = new ExportDataOption(); + ret.optionType = SupportedOptions.DatasetMetadataOnly; + return ret; + } + + public static ExportDataOption addPublicFilesOnly() { + ExportDataOption ret = new ExportDataOption(); + ret.optionType = SupportedOptions.PublicFilesOnly; + return ret; + } + + public boolean isDatasetMetadataOnly() { + return SupportedOptions.DatasetMetadataOnly.equals(optionType); + } + + public boolean isPublicFilesOnly() { + return SupportedOptions.PublicFilesOnly.equals(optionType); + } +} diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java index eefe0d7d828..54d98511d80 100644 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java +++ b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java @@ -38,7 +38,7 @@ public interface ExportDataProvider { * dataset-level metadata along with basic file metadata for each file * in the dataset. */ - JsonObject getDatasetORE(); + JsonObject getDatasetORE(ExportDataOption... options); /** * Dataverse is capable of extracting DDI-centric metadata from tabular @@ -53,7 +53,7 @@ public interface ExportDataProvider { * edu.harvard.iq.dataverse.util.json.JSONPrinter classes where this * output is used/generated (respectively). */ - JsonArray getDatasetFileDetails(); + JsonArray getDatasetFileDetails(ExportDataOption... options); /** * Similar to the above, but @@ -78,7 +78,7 @@ public interface ExportDataProvider { * a starting point for an Exporter if it simplifies your exporter * relative to using the JSON or OAI_ORE exports. */ - JsonObject getDatasetSchemaDotOrg(); + JsonObject getDatasetSchemaDotOrg(ExportDataOption... options); /** * @@ -88,7 +88,7 @@ public interface ExportDataProvider { * a starting point for an Exporter if it simplifies your exporter * relative to using the JSON or OAI_ORE exports. */ - String getDataCiteXml(); + String getDataCiteXml(ExportDataOption... options); /** * If an Exporter has specified a prerequisite format name via the @@ -108,13 +108,8 @@ public interface ExportDataProvider { * Exporter is configured to replace the internal ddi Exporter in * Dataverse. */ - default Optional getPrerequisiteInputStream() { + default Optional getPrerequisiteInputStream(ExportDataOption... options) { return Optional.empty(); } - - public enum ExportDataOption { - DatasetMetadataOnly, - PublicFilesOnly; - } - -} + + } From ec3f8aad2f144e4fb8ecc10680a93a18be3eb1aa Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 28 Aug 2025 09:40:43 -0400 Subject: [PATCH 120/634] #11710 fix release note --- doc/release-notes/11710-get-available-dataverses-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/11710-get-available-dataverses-api.md b/doc/release-notes/11710-get-available-dataverses-api.md index 6658d1d8fd5..ac33c581848 100644 --- a/doc/release-notes/11710-get-available-dataverses-api.md +++ b/doc/release-notes/11710-get-available-dataverses-api.md @@ -1,5 +1,5 @@ ### New API endpoint for retrieving a list of Dataverse Collections to which a given Dataset or Dataverse Collection may be linked --The end point also takes in a search term which currently must be the start of the collections' names. +-The end point also takes in a search term which currently must be part of the collections' names. -The user calling this API must have Link Dataset or Link Dataverse permission on the Dataverse Collections returned. -If the Collection has already been linked to the given Dataset or Collection, it will not be returned. From 239af1b84408371081140b4f63200ccc486ea686 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 28 Aug 2025 09:48:37 -0400 Subject: [PATCH 121/634] Update native-api.rst --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 0a208967f07..327e31124f6 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -735,7 +735,7 @@ Note: you must have "Add Dataset" permission in the given collection to invoke t List Dataverse Collections to which a given Dataset or Dataverse Collection may be linked ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The user may provide a search term to limit the list of Dataverse Collections returned. The search term will be compared to the name of the Dataverse Collections and must include the beginning of the Dataverse Collections' names. +The user may provide a search term to limit the list of Dataverse Collections returned. The search term will be compared to the name of the Dataverse Collections. The response is a JSON array of the ids, aliases, and names of the Dataverse collections to which a given Dataset or Dataverse Collection may be linked: For a given Dataverse Collection: From 85a3dccdc6df01c18dc333a40aac0dbe48b80937 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 28 Aug 2025 09:54:36 -0400 Subject: [PATCH 122/634] #11710 some code cleanup --- .../iq/dataverse/DataverseServiceBean.java | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 85d4179a6d9..bc91cb4e081 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -510,15 +510,15 @@ public List filterDataversesForLinking(String query, DataverseRequest List dataverseList = new ArrayList<>(); List results = filterDataversesByNamePattern(query); - + if (results == null || results.isEmpty()) { - return null; + return null; } - + Dataset linkedDataset = null; Dataverse linkedDataverse = null; List alreadyLinkeddv_ids; - + if ((dvo instanceof Dataset)) { linkedDataset = (Dataset) dvo; alreadyLinkeddv_ids = em.createNativeQuery("SELECT linkingdataverse_id FROM datasetlinkingdataverse WHERE dataset_id = " + linkedDataset.getId()).getResultList(); @@ -527,22 +527,22 @@ public List filterDataversesForLinking(String query, DataverseRequest alreadyLinkeddv_ids = em.createNativeQuery("SELECT linkingdataverse_id FROM dataverselinkingdataverse WHERE dataverse_id = " + linkedDataverse.getId()).getResultList(); } - List remove = new ArrayList<>(); + List remove = new ArrayList<>(); if (alreadyLinkeddv_ids != null && !alreadyLinkeddv_ids.isEmpty()) { alreadyLinkeddv_ids.stream().map((testDVId) -> this.find(testDVId)).forEachOrdered((removeIt) -> { remove.add(removeIt); }); } - - if (dvo instanceof Dataverse dataverse){ + + if (dvo instanceof Dataverse dataverse) { remove.add(dataverse); } - + for (Dataverse res : results) { if (!remove.contains(res)) { if ((linkedDataset != null && this.permissionService.requestOn(req, res).has(Permission.LinkDataset)) - || (linkedDataverse != null && this.permissionService.requestOn(req, res).has(Permission.LinkDataverse))) { + || (linkedDataverse != null && this.permissionService.requestOn(req, res).has(Permission.LinkDataverse))) { dataverseList.add(res); } } @@ -550,44 +550,43 @@ public List filterDataversesForLinking(String query, DataverseRequest return dataverseList; } - - - public List removeUnlinkableDataverses (List allWithPerms, DvObject dvo){ + + public List removeUnlinkableDataverses(List allWithPerms, DvObject dvo) { List dataverseList = new ArrayList<>(); Dataset linkedDataset = null; Dataverse linkedDataverse = null; List alreadyLinkeddv_ids; - + if ((dvo instanceof Dataset)) { linkedDataset = (Dataset) dvo; - alreadyLinkeddv_ids = em.createNativeQuery("SELECT linkingdataverse_id FROM datasetlinkingdataverse WHERE dataset_id = " + linkedDataset.getId()).getResultList(); + alreadyLinkeddv_ids = em.createNativeQuery("SELECT linkingdataverse_id FROM datasetlinkingdataverse WHERE dataset_id = " + linkedDataset.getId()).getResultList(); } else { linkedDataverse = (Dataverse) dvo; - alreadyLinkeddv_ids = em.createNativeQuery("SELECT linkingdataverse_id FROM dataverselinkingdataverse WHERE dataverse_id = " + linkedDataverse.getId()).getResultList(); + alreadyLinkeddv_ids = em.createNativeQuery("SELECT linkingdataverse_id FROM dataverselinkingdataverse WHERE dataverse_id = " + linkedDataverse.getId()).getResultList(); } - List remove = new ArrayList<>(); + List remove = new ArrayList<>(); if (alreadyLinkeddv_ids != null && !alreadyLinkeddv_ids.isEmpty()) { alreadyLinkeddv_ids.stream().map((testDVId) -> this.find(testDVId)).forEachOrdered((removeIt) -> { remove.add(removeIt); }); } - - if (dvo instanceof Dataverse dataverse){ + + if (dvo instanceof Dataverse dataverse) { remove.add(dataverse); - } else { + } else { //dataset is always owned by a dataverse - remove.add((Dataverse)dvo.getOwner()); + remove.add((Dataverse) dvo.getOwner()); } - + for (Dataverse res : allWithPerms) { if (!remove.contains(res)) { - dataverseList.add(res); + dataverseList.add(res); } } - - return dataverseList; + + return dataverseList; } From 286402f94bc6ebe5b8c7a04b765fd1c905cae787 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 28 Aug 2025 12:25:24 -0400 Subject: [PATCH 123/634] #11710 skip filter if search term empty --- .../harvard/iq/dataverse/api/Dataverses.java | 20 +++++++++++++------ .../iq/dataverse/api/DataversesIT.java | 9 ++++++++- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 512b7b00fab..97bc961c6b2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1785,8 +1785,7 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P DvObject dvObject = findDvoByIdAndTypeOrDie(dvIdtf, type, false); List dataversesForLinkingSearch = new ArrayList(); - dataversesForLinkingSearch = dataverseService.filterDataversesByNamePattern(searchTerm); - + AuthenticatedUser requestUser = (AuthenticatedUser) getRequestUser(crc); DataverseRequest dvReq = new DataverseRequest(requestUser, (IpAddress) null); List dataversesForLinking; @@ -1794,13 +1793,22 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P List mergedWithSearch = new ArrayList<>(); dataversesForLinking = dataverseService.removeUnlinkableDataverses(dataversesForLinking, dvObject); - if (!dataversesForLinkingSearch.isEmpty()) { - for (Dataverse dv : dataversesForLinking) { - if (dataversesForLinkingSearch.contains(dv)) { - mergedWithSearch.add(dv); + + //Only do search lookup if search term is there. Otherwise just include the collections based on perms + if (!searchTerm.isEmpty()) { + dataversesForLinkingSearch = dataverseService.filterDataversesByNamePattern(searchTerm); + if (!dataversesForLinkingSearch.isEmpty()) { + for (Dataverse dv : dataversesForLinking) { + if (dataversesForLinkingSearch.contains(dv)) { + mergedWithSearch.add(dv); + } } } + } else { + //search term empty then add all based on perms + mergedWithSearch.addAll(dataversesForLinking); } + JsonArrayBuilder dvBuilder = Json.createArrayBuilder(); if (!mergedWithSearch.isEmpty()) { for (Dataverse dv : mergedWithSearch) { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 64c88197494..5de3fbb6620 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -724,7 +724,14 @@ public void testGetLinkableDataverses(){ getLinkableDataversesForDataversePartial.prettyPrint(); getLinkableDataversesForDataversePartial.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data[0].alias", equalTo(dataverseAliasForLinking)); + .body("data[0].alias", equalTo(dataverseAliasForLinking)); + + //if I give a blank search term i should get the one that I have perms on + Response getLinkableDataversesForDataverseBlank = UtilIT.getLinkableDataverses("dataverse", dataverseAlias, apiToken, ""); + getLinkableDataversesForDataverseBlank.prettyPrint(); + getLinkableDataversesForDataverseBlank.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].alias", equalTo(dataverseAliasForLinking)); //Try with empty string search term From 1ab38bb559b2aaa366cffe881e630610f8cf3510 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 15 Aug 2025 13:16:10 -0400 Subject: [PATCH 124/634] base managed executor --- .../iq/dataverse/api/MakeDataCountApi.java | 178 +++++++++++------- 1 file changed, 107 insertions(+), 71 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java index 562fd7fcb81..72788f1d8e7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java @@ -27,7 +27,10 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; + +import jakarta.annotation.Resource; import jakarta.ejb.EJB; +import jakarta.enterprise.concurrent.ManagedExecutorService; import jakarta.json.Json; import jakarta.json.JsonArray; import jakarta.json.JsonArrayBuilder; @@ -62,6 +65,10 @@ public class MakeDataCountApi extends AbstractApiBean { @EJB SystemConfig systemConfig; + // Inject the managed executor service provided by the container + @Resource(name = "concurrent/CitationUpdateExecutor") + private ManagedExecutorService executorService; + /** * TODO: For each dataset, send the following: * @@ -141,89 +148,118 @@ public Response addUsageMetricsFromSushiReportAll(@QueryParam("reportOnDisk") St @POST @Path("{id}/updateCitationsForDataset") - public Response updateCitationsForDataset(@PathParam("id") String id) throws IOException { + public Response updateCitationsForDataset(@PathParam("id") String id) { try { - Dataset dataset = findDatasetOrDie(id); - GlobalId pid = dataset.getGlobalId(); - PidProvider pidProvider = PidUtil.getPidProvider(pid.getProviderId()); + // First validate that the dataset exists and has a valid DOI + final Dataset dataset = findDatasetOrDie(id); + final GlobalId pid = dataset.getGlobalId(); + final PidProvider pidProvider = PidUtil.getPidProvider(pid.getProviderId()); + // Only supported for DOIs and for DataCite DOI providers if(!DataCiteDOIProvider.TYPE.equals(pidProvider.getProviderType())) { return error(Status.BAD_REQUEST, "Only DataCite DOI providers are supported"); } - String persistentId = pid.toString(); - - // DataCite wants "doi=", not "doi:". - String authorityPlusIdentifier = persistentId.replaceFirst("doi:", ""); - // Request max page size and then loop to handle multiple pages - URL url = null; - try { - url = new URI(JvmSettings.DATACITE_REST_API_URL.lookup(pidProvider.getId()) + - "/events?doi=" + - authorityPlusIdentifier + - "&source=crossref&page[size]=1000&page[cursor]=1").toURL(); - } catch (URISyntaxException e) { - //Nominally this means a config error/ bad DATACITE_REST_API_URL for this provider - logger.warning("Unable to create URL for " + persistentId + ", pidProvider " + pidProvider.getId()); - return error(Status.INTERNAL_SERVER_ERROR, "Unable to create DataCite URL to retrieve citations."); - } - logger.fine("Retrieving Citations from " + url.toString()); - boolean nextPage = true; - JsonArrayBuilder dataBuilder = Json.createArrayBuilder(); - do { - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); - connection.setRequestMethod("GET"); - int status = connection.getResponseCode(); - if (status != 200) { - logger.warning("Failed to get citations from " + url.toString()); - connection.disconnect(); - return error(Status.fromStatusCode(status), "Failed to get citations from " + url.toString()); - } - JsonObject report; - try (InputStream inStream = connection.getInputStream()) { - report = JsonUtil.getJsonObject(inStream); - } finally { - connection.disconnect(); - } - JsonObject links = report.getJsonObject("links"); - JsonArray data = report.getJsonArray("data"); - Iterator iter = data.iterator(); - while (iter.hasNext()) { - dataBuilder.add(iter.next()); + + // Submit the task to the managed executor service + Future future = executorService.submit(() -> { + try { + processCitationUpdate(dataset, pid, pidProvider); + } catch (Exception e) { + logger.log(Level.SEVERE, "Error processing citation update for dataset " + id, e); } - if (links.containsKey("next")) { - try { - url = new URI(links.getString("next")).toURL(); - } catch (URISyntaxException e) { - logger.warning("Unable to create URL from DataCite response: " + links.getString("next")); - return error(Status.INTERNAL_SERVER_ERROR, "Unable to retrieve all results from DataCite"); - } - } else { - nextPage = false; - } - logger.fine("body of citation response: " + report.toString()); - } while (nextPage == true); - JsonArray allData = dataBuilder.build(); - List datasetExternalCitations = datasetExternalCitationsService.parseCitations(allData); - /* - * ToDo: If this is the only source of citations, we should remove all the existing ones for the dataset and repopulate them. - * As is, this call doesn't remove old citations if there are now none (legacy issue if we decide to stop counting certain types of citation - * as we've done for 'hasPart'). - * If there are some, this call individually checks each one and if a matching item exists, it removes it and adds it back. Faster and better to delete all and - * add the new ones. - */ - if (!datasetExternalCitations.isEmpty()) { - for (DatasetExternalCitations dm : datasetExternalCitations) { - datasetExternalCitationsService.save(dm); - } - } - + }); + JsonObjectBuilder output = Json.createObjectBuilder(); - output.add("citationCount", datasetExternalCitations.size()); + output.add("status", "queued"); + output.add("message", "Citation update for dataset " + id + " has been queued for processing"); return ok(output); } catch (WrappedResponse wr) { return wr.getResponse(); } } + + /** + * Process the citation update for a dataset + * This method contains the logic that was previously in updateCitationsForDataset + */ + private void processCitationUpdate(Dataset dataset, GlobalId pid, PidProvider pidProvider) throws IOException { + String persistentId = pid.asRawIdentifier(); + + // Request max page size and then loop to handle multiple pages + URL url = null; + try { + url = new URI(JvmSettings.DATACITE_REST_API_URL.lookup(pidProvider.getId()) + + "/events?doi=" + + persistentId + + "&source=crossref&page[size]=1000&page[cursor]=1").toURL(); + } catch (URISyntaxException e) { + //Nominally this means a config error/ bad DATACITE_REST_API_URL for this provider + logger.warning("Unable to create URL for " + persistentId + ", pidProvider " + pidProvider.getId()); + return; + } + + logger.fine("Retrieving Citations from " + url.toString()); + boolean nextPage = true; + JsonArrayBuilder dataBuilder = Json.createArrayBuilder(); + + do { + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("GET"); + int status = connection.getResponseCode(); + if (status != 200) { + logger.warning("Failed to get citations from " + url.toString()); + connection.disconnect(); + return; + } + + JsonObject report; + try (InputStream inStream = connection.getInputStream()) { + report = JsonUtil.getJsonObject(inStream); + } finally { + connection.disconnect(); + } + + JsonObject links = report.getJsonObject("links"); + JsonArray data = report.getJsonArray("data"); + Iterator iter = data.iterator(); + while (iter.hasNext()) { + dataBuilder.add(iter.next()); + } + + if (links.containsKey("next")) { + try { + url = new URI(links.getString("next")).toURL(); + } catch (URISyntaxException e) { + logger.warning("Unable to create URL from DataCite response: " + links.getString("next")); + return; + } + } else { + nextPage = false; + } + + logger.fine("body of citation response: " + report.toString()); + } while (nextPage == true); + + JsonArray allData = dataBuilder.build(); + List datasetExternalCitations = datasetExternalCitationsService.parseCitations(allData); + + /* + * ToDo: If this is the only source of citations, we should remove all the existing ones for the dataset and repopulate them. + * As is, this call doesn't remove old citations if there are now none (legacy issue if we decide to stop counting certain types of citation + * as we've done for 'hasPart'). + * If there are some, this call individually checks each one and if a matching item exists, it removes it and adds it back. Faster and better to delete all and + * add the new ones. + */ + if (!datasetExternalCitations.isEmpty()) { + for (DatasetExternalCitations dm : datasetExternalCitations) { + datasetExternalCitationsService.save(dm); + } + } + + logger.info("Citation update completed for dataset " + dataset.getId() + + " with " + datasetExternalCitations.size() + " citations"); + } + @GET @Path("{yearMonth}/processingState") public Response getProcessingState(@PathParam("yearMonth") String yearMonth) { From 3981933314e8c695f58315455c80f4f83506f01b Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 15 Aug 2025 13:38:02 -0400 Subject: [PATCH 125/634] add API_MDC_UPDATE_MIN_DELAY_MS --- .../iq/dataverse/api/MakeDataCountApi.java | 37 +++++++++++++++++++ .../iq/dataverse/settings/JvmSettings.java | 4 ++ 2 files changed, 41 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java index 72788f1d8e7..3bed917c789 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java @@ -25,6 +25,8 @@ import java.net.URL; import java.util.Iterator; import java.util.List; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; @@ -69,6 +71,9 @@ public class MakeDataCountApi extends AbstractApiBean { @Resource(name = "concurrent/CitationUpdateExecutor") private ManagedExecutorService executorService; + // Track the last execution time to implement rate limiting during Citation updates + private static final AtomicLong lastExecutionTime = new AtomicLong(0); + /** * TODO: For each dataset, send the following: * @@ -163,7 +168,14 @@ public Response updateCitationsForDataset(@PathParam("id") String id) { // Submit the task to the managed executor service Future future = executorService.submit(() -> { try { + // Apply rate limiting if enabled + applyRateLimit(); + + // Process the citation update processCitationUpdate(dataset, pid, pidProvider); + + // Update the last execution time after processing + lastExecutionTime.set(System.currentTimeMillis()); } catch (Exception e) { logger.log(Level.SEVERE, "Error processing citation update for dataset " + id, e); } @@ -178,6 +190,31 @@ public Response updateCitationsForDataset(@PathParam("id") String id) { } } + /** + * Apply rate limiting by waiting if necessary + */ + private void applyRateLimit() { + // Check if rate limiting is enabled + long minDelay = JvmSettings.API_MDC_UPDATE_MIN_DELAY_MS.lookupOptional(Long.class).orElse(0l); + + // Calculate how long to wait + long lastExecution = lastExecutionTime.get(); + long currentTime = System.currentTimeMillis(); + long elapsedTime = currentTime - lastExecution; + + // If not enough time has passed since the last execution, wait + if (lastExecution > 0 && elapsedTime < minDelay) { + long waitTime = minDelay - elapsedTime; + logger.fine("Rate limiting: waiting " + waitTime + " ms before processing next citation update"); + try { + Thread.sleep(waitTime); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warning("Rate limiting sleep interrupted: " + e.getMessage()); + } + } + } + /** * Process the citation update for a dataset * This method contains the logic that was previously in updateCitationsForDataset diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index 53dff244ae1..87123801a3e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -93,6 +93,10 @@ public enum JvmSettings { API_BLOCKED_ENDPOINTS(SCOPE_API_BLOCKED, "endpoints"), API_BLOCKED_POLICY(SCOPE_API_BLOCKED, "policy"), API_BLOCKED_KEY(SCOPE_API_BLOCKED, "key"), + // API: MDC Citation updates + SCOPE_API_MDC(SCOPE_API, "mdc"), + API_MDC_UPDATE_MIN_DELAY_MS(SCOPE_API_MDC, "min-delay-ms"), + // SIGNPOSTING SETTINGS SCOPE_SIGNPOSTING(PREFIX, "signposting"), From 973ab872e203085aee6be078eba85706ac0ac601 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 15 Aug 2025 13:47:43 -0400 Subject: [PATCH 126/634] filter out hasPart etc earlier --- .../iq/dataverse/api/MakeDataCountApi.java | 22 +++++++++++++++++-- .../DatasetExternalCitationsServiceBean.java | 4 ++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java index 3bed917c789..64309886dd0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java @@ -196,7 +196,9 @@ public Response updateCitationsForDataset(@PathParam("id") String id) { private void applyRateLimit() { // Check if rate limiting is enabled long minDelay = JvmSettings.API_MDC_UPDATE_MIN_DELAY_MS.lookupOptional(Long.class).orElse(0l); - + if(minDelay ==0) { + return; + } // Calculate how long to wait long lastExecution = lastExecutionTime.get(); long currentTime = System.currentTimeMillis(); @@ -260,12 +262,28 @@ private void processCitationUpdate(Dataset dataset, GlobalId pid, PidProvider pi JsonArray data = report.getJsonArray("data"); Iterator iter = data.iterator(); while (iter.hasNext()) { - dataBuilder.add(iter.next()); + JsonValue citationValue = iter.next(); + JsonObject citation = (JsonObject) citationValue; + + // Filter out relations we don't use (e.g. hasPart) to lower memory req. with many files + if (citation.containsKey("attributes")) { + JsonObject attributes = citation.getJsonObject("attributes"); + if (attributes.containsKey("relation-type-id")) { + String relationshipType = attributes.getString("relation-type-id"); + + // Only add citations with relationship types we care about + if (DatasetExternalCitationsServiceBean.inboundRelationships.contains(relationshipType) || + DatasetExternalCitationsServiceBean.outboundRelationships.contains(relationshipType)) { + dataBuilder.add(citationValue); + } + } + } } if (links.containsKey("next")) { try { url = new URI(links.getString("next")).toURL(); + applyRateLimit(); } catch (URISyntaxException e) { logger.warning("Unable to create URL from DataCite response: " + links.getString("next")); return; diff --git a/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetExternalCitationsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetExternalCitationsServiceBean.java index fa56432cc3c..fa87926210f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetExternalCitationsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetExternalCitationsServiceBean.java @@ -39,13 +39,13 @@ public class DatasetExternalCitationsServiceBean implements java.io.Serializable DatasetServiceBean datasetService; //Array of relationship types that are considered to be citations - static ArrayList inboundRelationships = new ArrayList( + public static ArrayList inboundRelationships = new ArrayList( Arrays.asList( "cites", "references", "supplements", "is-supplement-to")); - static ArrayList outboundRelationships = new ArrayList( + public static ArrayList outboundRelationships = new ArrayList( Arrays.asList( "is-cited-by", "is-referenced-by", From 5f6b076ab248e69f414b4df0a64508f9598c0417 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 15 Aug 2025 15:06:54 -0400 Subject: [PATCH 127/634] cleanup logging/exception handling --- .../iq/dataverse/api/MakeDataCountApi.java | 147 ++++++++++-------- 1 file changed, 80 insertions(+), 67 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java index 64309886dd0..6de0e86a254 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java @@ -172,10 +172,16 @@ public Response updateCitationsForDataset(@PathParam("id") String id) { applyRateLimit(); // Process the citation update - processCitationUpdate(dataset, pid, pidProvider); + boolean success = processCitationUpdate(dataset, pid, pidProvider); // Update the last execution time after processing lastExecutionTime.set(System.currentTimeMillis()); + + if (success) { + logger.fine("Successfully processed citation update for dataset " + id); + } else { + logger.warning("Failed to process citation update for dataset " + id); + } } catch (Exception e) { logger.log(Level.SEVERE, "Error processing citation update for dataset " + id, e); } @@ -220,8 +226,9 @@ private void applyRateLimit() { /** * Process the citation update for a dataset * This method contains the logic that was previously in updateCitationsForDataset + * @return true if processing was successful, false otherwise */ - private void processCitationUpdate(Dataset dataset, GlobalId pid, PidProvider pidProvider) throws IOException { + private boolean processCitationUpdate(Dataset dataset, GlobalId pid, PidProvider pidProvider) { String persistentId = pid.asRawIdentifier(); // Request max page size and then loop to handle multiple pages @@ -231,88 +238,94 @@ private void processCitationUpdate(Dataset dataset, GlobalId pid, PidProvider pi "/events?doi=" + persistentId + "&source=crossref&page[size]=1000&page[cursor]=1").toURL(); - } catch (URISyntaxException e) { + } catch (URISyntaxException | MalformedURLException e) { //Nominally this means a config error/ bad DATACITE_REST_API_URL for this provider logger.warning("Unable to create URL for " + persistentId + ", pidProvider " + pidProvider.getId()); - return; + return false; } logger.fine("Retrieving Citations from " + url.toString()); boolean nextPage = true; JsonArrayBuilder dataBuilder = Json.createArrayBuilder(); - do { - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); - connection.setRequestMethod("GET"); - int status = connection.getResponseCode(); - if (status != 200) { - logger.warning("Failed to get citations from " + url.toString()); - connection.disconnect(); - return; - } - - JsonObject report; - try (InputStream inStream = connection.getInputStream()) { - report = JsonUtil.getJsonObject(inStream); - } finally { - connection.disconnect(); - } - - JsonObject links = report.getJsonObject("links"); - JsonArray data = report.getJsonArray("data"); - Iterator iter = data.iterator(); - while (iter.hasNext()) { - JsonValue citationValue = iter.next(); - JsonObject citation = (JsonObject) citationValue; + try { + do { + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("GET"); + int status = connection.getResponseCode(); + if (status != 200) { + logger.warning("Failed to get citations from " + url.toString()); + connection.disconnect(); + return false; + } - // Filter out relations we don't use (e.g. hasPart) to lower memory req. with many files - if (citation.containsKey("attributes")) { - JsonObject attributes = citation.getJsonObject("attributes"); - if (attributes.containsKey("relation-type-id")) { - String relationshipType = attributes.getString("relation-type-id"); - - // Only add citations with relationship types we care about - if (DatasetExternalCitationsServiceBean.inboundRelationships.contains(relationshipType) || - DatasetExternalCitationsServiceBean.outboundRelationships.contains(relationshipType)) { - dataBuilder.add(citationValue); + JsonObject report; + try (InputStream inStream = connection.getInputStream()) { + report = JsonUtil.getJsonObject(inStream); + } finally { + connection.disconnect(); + } + + JsonObject links = report.getJsonObject("links"); + JsonArray data = report.getJsonArray("data"); + Iterator iter = data.iterator(); + while (iter.hasNext()) { + JsonValue citationValue = iter.next(); + JsonObject citation = (JsonObject) citationValue; + + // Filter out relations we don't use (e.g. hasPart) to lower memory req. with many files + if (citation.containsKey("attributes")) { + JsonObject attributes = citation.getJsonObject("attributes"); + if (attributes.containsKey("relation-type-id")) { + String relationshipType = attributes.getString("relation-type-id"); + + // Only add citations with relationship types we care about + if (DatasetExternalCitationsServiceBean.inboundRelationships.contains(relationshipType) || + DatasetExternalCitationsServiceBean.outboundRelationships.contains(relationshipType)) { + dataBuilder.add(citationValue); + } } } } - } + + if (links.containsKey("next")) { + try { + url = new URI(links.getString("next")).toURL(); + applyRateLimit(); + } catch (URISyntaxException e) { + logger.warning("Unable to create URL from DataCite response: " + links.getString("next")); + return false; + } + } else { + nextPage = false; + } + + logger.fine("body of citation response: " + report.toString()); + } while (nextPage == true); - if (links.containsKey("next")) { - try { - url = new URI(links.getString("next")).toURL(); - applyRateLimit(); - } catch (URISyntaxException e) { - logger.warning("Unable to create URL from DataCite response: " + links.getString("next")); - return; + JsonArray allData = dataBuilder.build(); + List datasetExternalCitations = datasetExternalCitationsService.parseCitations(allData); + + /* + * ToDo: If this is the only source of citations, we should remove all the existing ones for the dataset and repopulate them. + * As is, this call doesn't remove old citations if there are now none (legacy issue if we decide to stop counting certain types of citation + * as we've done for 'hasPart'). + * If there are some, this call individually checks each one and if a matching item exists, it removes it and adds it back. Faster and better to delete all and + * add the new ones. + */ + if (!datasetExternalCitations.isEmpty()) { + for (DatasetExternalCitations dm : datasetExternalCitations) { + datasetExternalCitationsService.save(dm); } - } else { - nextPage = false; } - logger.fine("body of citation response: " + report.toString()); - } while (nextPage == true); - - JsonArray allData = dataBuilder.build(); - List datasetExternalCitations = datasetExternalCitationsService.parseCitations(allData); - - /* - * ToDo: If this is the only source of citations, we should remove all the existing ones for the dataset and repopulate them. - * As is, this call doesn't remove old citations if there are now none (legacy issue if we decide to stop counting certain types of citation - * as we've done for 'hasPart'). - * If there are some, this call individually checks each one and if a matching item exists, it removes it and adds it back. Faster and better to delete all and - * add the new ones. - */ - if (!datasetExternalCitations.isEmpty()) { - for (DatasetExternalCitations dm : datasetExternalCitations) { - datasetExternalCitationsService.save(dm); - } + logger.fine("Citation update completed for dataset " + dataset.getId() + + " with " + datasetExternalCitations.size() + " citations"); + return true; + } catch (IOException e) { + logger.log(Level.WARNING, "Error processing citation update for dataset " + dataset.getId(), e); + return false; } - - logger.info("Citation update completed for dataset " + dataset.getId() + - " with " + datasetExternalCitations.size() + " citations"); } @GET From a96dffb0678485601e50324d7cb701e08836d350 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 15 Aug 2025 16:05:33 -0400 Subject: [PATCH 128/634] handle queue full error --- .../iq/dataverse/api/MakeDataCountApi.java | 62 +++++++++++-------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java index 6de0e86a254..ca4f55da822 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java @@ -26,6 +26,7 @@ import java.util.Iterator; import java.util.List; import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; @@ -159,38 +160,45 @@ public Response updateCitationsForDataset(@PathParam("id") String id) { final Dataset dataset = findDatasetOrDie(id); final GlobalId pid = dataset.getGlobalId(); final PidProvider pidProvider = PidUtil.getPidProvider(pid.getProviderId()); - + // Only supported for DOIs and for DataCite DOI providers - if(!DataCiteDOIProvider.TYPE.equals(pidProvider.getProviderType())) { + if (!DataCiteDOIProvider.TYPE.equals(pidProvider.getProviderType())) { return error(Status.BAD_REQUEST, "Only DataCite DOI providers are supported"); } - + // Submit the task to the managed executor service - Future future = executorService.submit(() -> { - try { - // Apply rate limiting if enabled - applyRateLimit(); - - // Process the citation update - boolean success = processCitationUpdate(dataset, pid, pidProvider); - - // Update the last execution time after processing - lastExecutionTime.set(System.currentTimeMillis()); - - if (success) { - logger.fine("Successfully processed citation update for dataset " + id); - } else { - logger.warning("Failed to process citation update for dataset " + id); + Future future; + try { + future = executorService.submit(() -> { + try { + // Apply rate limiting if enabled + applyRateLimit(); + + // Process the citation update + boolean success = processCitationUpdate(dataset, pid, pidProvider); + + // Update the last execution time after processing + lastExecutionTime.set(System.currentTimeMillis()); + + if (success) { + logger.fine("Successfully processed citation update for dataset " + id); + } else { + logger.warning("Failed to process citation update for dataset " + id); + } + } catch (Exception e) { + logger.log(Level.SEVERE, "Error processing citation update for dataset " + id, e); } - } catch (Exception e) { - logger.log(Level.SEVERE, "Error processing citation update for dataset " + id, e); - } - }); - - JsonObjectBuilder output = Json.createObjectBuilder(); - output.add("status", "queued"); - output.add("message", "Citation update for dataset " + id + " has been queued for processing"); - return ok(output); + }); + + JsonObjectBuilder output = Json.createObjectBuilder(); + output.add("status", "queued"); + output.add("message", "Citation update for dataset " + id + " has been queued for processing"); + return ok(output); + } catch (RejectedExecutionException ree) { + logger.warning("Citation update for dataset " + id + " was rejected: Queue is full"); + return error(Status.SERVICE_UNAVAILABLE, + "Citation update service is currently at capacity. Please try again later."); + } } catch (WrappedResponse wr) { return wr.getResponse(); } From 5d8095b190e8be29c9bdea4d4797d29123de9d59 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 15 Aug 2025 16:05:48 -0400 Subject: [PATCH 129/634] update script for asynch api call --- conf/mdc/counter_weekly.sh | 92 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 conf/mdc/counter_weekly.sh diff --git a/conf/mdc/counter_weekly.sh b/conf/mdc/counter_weekly.sh new file mode 100644 index 00000000000..67cb5df2af2 --- /dev/null +++ b/conf/mdc/counter_weekly.sh @@ -0,0 +1,92 @@ +#!/bin/sh +#counter_weekly.sh + +# This script iterates through all published Datasets in all Dataverses and calls the Make Data Count API to update their citations from DataCite +# Note: Requires curl and jq for parsing JSON responses form curl + +# A recursive method to process each Dataverse +processDV () { +echo "Processing Dataverse ID#: $1" + +#Call the Dataverse API to get the contents of the Dataverse (without credentials, this will only list published datasets and dataverses +DVCONTENTS=$(curl -s http://localhost:8080/api/dataverses/$1/contents) + +# Iterate over all datasets, pulling the value of their DOIs (as part of the persistentUrl) from the json returned +for subds in $(echo "${DVCONTENTS}" | jq -r '.data[] | select(.type == "dataset") | .persistentUrl'); do + +#The authority/identifier are preceded by a protocol/host, i.e. https://doi.org/ +DOI=`expr "$subds" : '.*:\/\/\doi\.org\/\(.*\)'` + +# Call the Dataverse API for this dataset and capture both the response and HTTP status code +HTTP_RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "http://localhost:8080/api/admin/makeDataCount/:persistentId/updateCitationsForDataset?persistentId=doi:$DOI") + +# Extract the HTTP status code from the last line +HTTP_STATUS=$(echo "$HTTP_RESPONSE" | tail -n1) +# Extract the response body (everything except the last line) +RESPONSE_BODY=$(echo "$HTTP_RESPONSE" | sed '$d') + +# Check the HTTP status code and report accordingly +case $HTTP_STATUS in + 200) + # Successfully queued + # Extract status from the nested data object + STATUS=$(echo "$RESPONSE_BODY" | jq -r '.data.status') + + # Extract message from the nested data object + if echo "$RESPONSE_BODY" | jq -e '.data.message' > /dev/null 2>&1 && [ "$(echo "$RESPONSE_BODY" | jq -r '.data.message')" != "null" ]; then + MESSAGE=$(echo "$RESPONSE_BODY" | jq -r '.data.message') + echo "[SUCCESS] doi:$DOI - $STATUS: $MESSAGE" + else + # If message is missing or null, just show the status + echo "[SUCCESS] doi:$DOI - $STATUS: Citation update queued" + fi + ;; + 400) + # Bad request + if echo "$RESPONSE_BODY" | jq -e '.message' > /dev/null 2>&1; then + ERROR=$(echo "$RESPONSE_BODY" | jq -r '.message') + echo "[ERROR 400] doi:$DOI - Bad request: $ERROR" + else + echo "[ERROR 400] doi:$DOI - Bad request" + fi + ;; + 404) + # Not found + if echo "$RESPONSE_BODY" | jq -e '.message' > /dev/null 2>&1; then + ERROR=$(echo "$RESPONSE_BODY" | jq -r '.message') + echo "[ERROR 404] doi:$DOI - Not found: $ERROR" + else + echo "[ERROR 404] doi:$DOI - Not found" + fi + ;; + 503) + # Service unavailable (queue full) + if echo "$RESPONSE_BODY" | jq -e '.message' > /dev/null 2>&1; then + ERROR=$(echo "$RESPONSE_BODY" | jq -r '.message') + echo "[ERROR 503] doi:$DOI - Service unavailable: $ERROR" + elif echo "$RESPONSE_BODY" | jq -e '.data.message' > /dev/null 2>&1; then + ERROR=$(echo "$RESPONSE_BODY" | jq -r '.data.message') + echo "[ERROR 503] doi:$DOI - Service unavailable: $ERROR" + else + echo "[ERROR 503] doi:$DOI - Service unavailable: Queue is full" + fi + ;; + *) + # Other error + echo "[ERROR $HTTP_STATUS] doi:$DOI - Unexpected error" + echo "Response: $RESPONSE_BODY" + ;; +esac + +done + +# Now iterate over any child Dataverses and recursively process them +for subdv in $(echo "${DVCONTENTS}" | jq -r '.data[] | select(.type == "dataverse") | .id'); do +echo $subdv +processDV $subdv +done + +} + +# Call the function on the root dataverse to start processing +processDV 1 \ No newline at end of file From 75d99d09158c27a7cbf360c52d7a3611cff7809d Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 29 Aug 2025 13:16:54 -0400 Subject: [PATCH 130/634] another experimental iteration of the updated export data spi. #11766 --- .../io/gdcc/spi/export/ExportDataContext.java | 13 +++++++ .../io/gdcc/spi/export/ExportDataOption.java | 1 + .../gdcc/spi/export/ExportDataProvider.java | 36 +++++++++++-------- .../java/io/gdcc/spi/export/Exporter.java | 14 +++++++- 4 files changed, 49 insertions(+), 15 deletions(-) create mode 100644 modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataContext.java diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataContext.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataContext.java new file mode 100644 index 00000000000..8776f2047ba --- /dev/null +++ b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataContext.java @@ -0,0 +1,13 @@ +/* + * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license + * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template + */ +package io.gdcc.spi.export; + +/** + * + * @author landreev + */ +public class ExportDataContext { + +} diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataOption.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataOption.java index 1c447cc53e3..69f813f83ce 100644 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataOption.java +++ b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataOption.java @@ -8,6 +8,7 @@ * options going forward with minimal or no changes to the existing code in * export plugins. */ +@Deprecated public class ExportDataOption { public enum SupportedOptions { diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java index 54d98511d80..4197d978e79 100644 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java +++ b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java @@ -21,13 +21,14 @@ public interface ExportDataProvider { * OAI_ORE export are the only two that provide 'complete' * dataset-level metadata along with basic file metadata for each file * in the dataset. - * @param options - optional argument(s). needs to support ExportDataOption.DatasetMetadataOnly: - * in a situation where we need to generate a format like DC, - * that has no use for file-level metadata, it makes sense to - * skip retrieving and formatting it, since there can be quite a few + * @param context - supplies optional parameters. Needs to support + * context.isDatasetMetadataOnly(). In a situation where we + * need to generate a format like DC that has no use for the + * file-level metadata, it makes sense to skip retrieving and + * formatting it, since there can be a very large number of * files in a dataset. */ - JsonObject getDatasetJson(ExportDataOption... options); + JsonObject getDatasetJson(ExportDataContext... context); /** * @@ -37,8 +38,9 @@ public interface ExportDataProvider { * @apiNote - THis, and the JSON format are the only two that provide complete * dataset-level metadata along with basic file metadata for each file * in the dataset. + * @param context - supplies optional parameters. */ - JsonObject getDatasetORE(ExportDataOption... options); + JsonObject getDatasetORE(ExportDataContext... context); /** * Dataverse is capable of extracting DDI-centric metadata from tabular @@ -52,8 +54,9 @@ public interface ExportDataProvider { * edu.harvard.iq.dataverse.export.DDIExporter and the @see * edu.harvard.iq.dataverse.util.json.JSONPrinter classes where this * output is used/generated (respectively). + * @param context - supplies optional parameters. */ - JsonArray getDatasetFileDetails(ExportDataOption... options); + JsonArray getDatasetFileDetails(ExportDataContext... context); /** * Similar to the above, but @@ -61,13 +64,15 @@ public interface ExportDataProvider { * b) provides an option for retrieving this stuff in batches * c) provides an option for skipping restricted/embargoed etc. files. * Intended for datasets with massive numbers of tabular files and datavariables. - * @param offset (can be null) - * @param length (can be null) - * @param options (optional) current use case is ExportDataOption.PublicFilesOnly; + * @param context - supplies optional parameters. + * current (2.1.0) known use cases: + * context.isPublicFilesOnly(); + * context.getOffset(); + * context.getLength(); * @return json array containing the datafile/filemetadata->datatable->datavariable metadata * @throws ExportException */ - JsonArray getTabularDataDetails(Integer offset, Integer length, ExportDataOption ... options) throws ExportException; + JsonArray getTabularDataDetails(ExportDataContext ... context) throws ExportException; /** * @@ -77,8 +82,9 @@ public interface ExportDataProvider { * @apiNote - as this metadata export is not complete, it should only be used as * a starting point for an Exporter if it simplifies your exporter * relative to using the JSON or OAI_ORE exports. + * @param context - supplies optional parameters. */ - JsonObject getDatasetSchemaDotOrg(ExportDataOption... options); + JsonObject getDatasetSchemaDotOrg(ExportDataContext... context); /** * @@ -87,8 +93,9 @@ public interface ExportDataProvider { * @apiNote - as this metadata export is not complete, it should only be used as * a starting point for an Exporter if it simplifies your exporter * relative to using the JSON or OAI_ORE exports. + * @param context - supplies optional parameters. */ - String getDataCiteXml(ExportDataOption... options); + String getDataCiteXml(ExportDataContext... context); /** * If an Exporter has specified a prerequisite format name via the @@ -107,8 +114,9 @@ public interface ExportDataProvider { * malfunction, e.g. if you depend on format "ddi" and a third party * Exporter is configured to replace the internal ddi Exporter in * Dataverse. + * @param context - supplies optional parameters. */ - default Optional getPrerequisiteInputStream(ExportDataOption... options) { + default Optional getPrerequisiteInputStream(ExportDataContext... context) { return Optional.empty(); } diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/Exporter.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/Exporter.java index 1338a3c9734..f91a2504f74 100644 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/Exporter.java +++ b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/Exporter.java @@ -84,7 +84,19 @@ public interface Exporter { default Optional getPrerequisiteFormatName() { return Optional.empty(); } - + + /** + * Most metadata formats do not require tabular metadata to be generated. + * (the 2 known cases that encode variable-level information, as of Dataverse 6.7, + * are rich DDI and Croissant). In order to serve it more efficiently, + * the data provider may need to be initialized in a special way (with access + * to EJBs that in most cases isn't needed), so it may help to know + * programmatically whether it's required for this format. + * @return Optional + */ + default Optional isNeedsTabularMetadata() { + return Optional.of(false); + } /** * Harvestable Exporters will be available as options in Dataverse's Harvesting mechanism. From cb06d8962863e9d2ffdc279637d92969cc2a1c79 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 29 Aug 2025 13:42:31 -0400 Subject: [PATCH 131/634] the new context object (was missing from the previous commit in error) #11766 --- .../io/gdcc/spi/export/ExportDataContext.java | 56 +++++++++++++++++-- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataContext.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataContext.java index 8776f2047ba..9478d39c4c2 100644 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataContext.java +++ b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataContext.java @@ -1,13 +1,61 @@ -/* - * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license - * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template - */ package io.gdcc.spi.export; /** * * @author landreev + * Provides an optional mechanism for defining various data retrieval options + * for the export subsystem in a way that should allow us adding support for + * more options going forward with minimal or no changes to the already + * implemented export plugins. */ public class ExportDataContext { + private boolean datasetMetadataOnly = false; + private boolean publicFilesOnly = false; + private Integer offset = null; + private Integer length = null; + + private ExportDataContext() { + + } + + public static ExportDataContext context() { + ExportDataContext context = new ExportDataContext(); + return context; + } + + public ExportDataContext withDatasetMetadataOnly() { + this.datasetMetadataOnly = true; + return this; + } + + public ExportDataContext withPublicFilesOnly() { + this.publicFilesOnly = true; + return this; + } + + public ExportDataContext withOffset(Integer offset) { + this.offset = offset; + return this; + } + + public ExportDataContext withLength(Integer length) { + this.length = length; + return this; + } + + public boolean isDatasetMetadataOnly() { + return datasetMetadataOnly; + } + + public boolean isPublicFilesOnly() { + return publicFilesOnly; + } + + public Integer getOffset() { + return offset; + } + public Integer getLength() { + return length; + } } From ba61c09459fdc9e994d30359b81031f001fe8324 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Aug 2025 14:35:50 -0400 Subject: [PATCH 132/634] release note --- doc/release-notes/11777-MDC-citation-api-improvement.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/release-notes/11777-MDC-citation-api-improvement.md diff --git a/doc/release-notes/11777-MDC-citation-api-improvement.md b/doc/release-notes/11777-MDC-citation-api-improvement.md new file mode 100644 index 00000000000..9441e9e0f44 --- /dev/null +++ b/doc/release-notes/11777-MDC-citation-api-improvement.md @@ -0,0 +1,7 @@ +The /api/admin/makeDataCount/{id}/updateCitationsForDataset endpoint, which allows citations for a dataset to be retrieved from DataCite, is often called periodically for all datasets. However, allowing calls for many datasets to be processed in parallel can cause performance problems in Dataverse and/or cause calls to DataCite to fail due to rate limiting. The existing implementation was also inefficient w.r.t. memory use when used on datasets with many (>~1K) files. This release configures Dataverse to queue calls to this api, processes them serially, adds optional throttling to avoid hitting DataCite rate limits and improves memory use. + +New optional MPConfig setting: + +dataverse.api.mdc.min-delay-ms - number of milliseconds to wait between calls to DataCite. A value of ~100 should conservatively address DataCite's current 3000/5 minute limit. A value of 250 may be required for their test service. + +Backward compatibility: This api call is now asynchronous and will return an OK response when the call is queued or a 503 if the queue is full. \ No newline at end of file From 71321b995f7667f53c3eb2c8a4aec1e94d3cecd2 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Aug 2025 14:36:09 -0400 Subject: [PATCH 133/634] switch to app executor service --- src/main/webapp/WEB-INF/glassfish-resources.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/webapp/WEB-INF/glassfish-resources.xml b/src/main/webapp/WEB-INF/glassfish-resources.xml index 3fbbf4c3586..74af3be42ce 100644 --- a/src/main/webapp/WEB-INF/glassfish-resources.xml +++ b/src/main/webapp/WEB-INF/glassfish-resources.xml @@ -11,4 +11,14 @@ + + + + + + + + \ No newline at end of file From 867af5e08398f88a0fd211a9e1e1eb3e145ee634 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Aug 2025 14:46:05 -0400 Subject: [PATCH 134/634] docs --- .../source/admin/make-data-count.rst | 2 ++ doc/sphinx-guides/source/api/changelog.rst | 2 +- doc/sphinx-guides/source/installation/config.rst | 16 ++++++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/admin/make-data-count.rst b/doc/sphinx-guides/source/admin/make-data-count.rst index 0103a6f9e38..f8ffa7bb084 100644 --- a/doc/sphinx-guides/source/admin/make-data-count.rst +++ b/doc/sphinx-guides/source/admin/make-data-count.rst @@ -166,6 +166,8 @@ The example :download:`counter_weekly.sh <../_static/util/counter_weekly.sh>` wi Citations will be retrieved for each published dataset and recorded in the your Dataverse installation's database. +Note that the :ref:`dataverse.api.mdc.min-delay-ms` setting can be used to avoid getting rate-limit errors from DataCite. + For how to get the citations out of your Dataverse installation, see "Retrieving Citations for a Dataset" under :ref:`Dataset Metrics ` in the :doc:`/api/native-api` section of the API Guide. Please note that while the Dataverse Software has a metadata field for "Related Dataset" this information is not currently sent as a citation to Crossref. diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 16157459220..08e8620ba13 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -13,7 +13,7 @@ v6.8 - For POST /api/files/{id}/metadata passing an empty string ("description":"") or array ("categories":[]) will no longer be ignored. Empty fields will now clear out the values in the file's metadata. To ignore the fields simply do not include them in the JSON string. - For PUT /api/datasets/{id}/editMetadata the query parameter "sourceInternalVersionNumber" has been removed and replaced with "sourceLastUpdateTime" to verify that the data being edited hasn't been modified and isn't stale. - For GET /api/dataverses/$dataverse-alias/links the Json response has changed breaking the backward compatibility of the API. - +- The POST /api/admin/makeDataCount/{id}/updateCitationsForDataset processing is now asynchronous and the response no longer includes the number of citations. The response can be OK if the request is queued or 503 if the queue is full (default queue size is 1000). v6.7 ---- diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index d2eff275392..a8e6129c501 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3729,6 +3729,22 @@ Example: Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_CORS_HEADERS_EXPOSE``. + +.. _dataverse.api.mdc.min-delay-ms: + +dataverse.api.mdc.min-delay-ms +++++++++++++++++++++++++++++++ + +Minimum delay in milliseconds between Make Data Count (MDC) API requests from the /api/admin/makeDataCount/{id}/updateCitationsForDataset api. +This setting helps prevent overloading the MDC service by enforcing a minimum time interval between consecutive requests. +If a request arrives before this interval has elapsed since the previous request, it will be rate-limited. + +Default: ``0`` (no delay enforced) + +Example: ``dataverse.api.mdc.min-delay-ms=100`` (enforces a minimum 100ms delay between MDC API requests) + +Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_API_MDC_MIN_DELAY_MS``. + .. _feature-flags: Feature Flags From 85fcbac7a05d652360fd9fac3ac5ccd8a06627f5 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Aug 2025 15:10:54 -0400 Subject: [PATCH 135/634] missing blank line --- doc/sphinx-guides/source/api/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 08e8620ba13..4c91a63f86d 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -14,6 +14,7 @@ v6.8 - For PUT /api/datasets/{id}/editMetadata the query parameter "sourceInternalVersionNumber" has been removed and replaced with "sourceLastUpdateTime" to verify that the data being edited hasn't been modified and isn't stale. - For GET /api/dataverses/$dataverse-alias/links the Json response has changed breaking the backward compatibility of the API. - The POST /api/admin/makeDataCount/{id}/updateCitationsForDataset processing is now asynchronous and the response no longer includes the number of citations. The response can be OK if the request is queued or 503 if the queue is full (default queue size is 1000). + v6.7 ---- From fcb8b9d97b6d3eab94f656896bcd65eba5e16a41 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Aug 2025 15:22:23 -0400 Subject: [PATCH 136/634] check for string --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index baf8302437d..1bd0f4a1adc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -58,6 +58,7 @@ import edu.harvard.iq.dataverse.util.xml.XmlWriterUtil; import jakarta.enterprise.inject.spi.CDI; import jakarta.json.JsonObject; +import jakarta.json.JsonValue.ValueType; public class XmlMetadataTemplate { @@ -621,7 +622,8 @@ private void writeEntityElements(XMLStreamWriter xmlw, String elementName, Strin if (externalIdentifier.isValidIdentifier(orgName)) { isROR = true; JsonObject jo = getExternalVocabularyValue(orgName); - if (jo != null) { + // Some ext. cvv configs store a JsonArray of multiple objects/values. In such cases, we'll leave orgName blank + if (jo != null && jo.getValueType() == ValueType.STRING) { orgName = jo.getString("termName"); } } From 05b68c621bf368ce3363151cbef4f33c4874396c Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 26 Aug 2025 16:58:03 -0400 Subject: [PATCH 137/634] and another --- src/main/java/edu/harvard/iq/dataverse/export/ExportService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java index b98f88e386f..e7bcf17d44b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java @@ -312,6 +312,7 @@ public void exportAllFormats(Dataset dataset) throws ExportException { } catch (ServiceConfigurationError serviceError) { throw new ExportException("Service configuration error during export. " + serviceError.getMessage()); } catch (RuntimeException e) { + e.printStackTrace(); logger.log(Level.FINE, e.getMessage(), e); throw new ExportException( "Unknown runtime exception exporting metadata. " + (e.getMessage() == null ? "" : e.getMessage())); From 68ad098c5dae9c63107dea70ffcf05316668949d Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 19 Aug 2025 16:55:14 -0400 Subject: [PATCH 138/634] preserve curation statuses during update-current and add blank status as in finalize publication --- .../CuratePublishedDatasetVersionCommand.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index 3629432b7e4..21139ac3fbc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -15,6 +15,7 @@ import edu.harvard.iq.dataverse.DatasetField; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; +import edu.harvard.iq.dataverse.CurationStatus; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.RoleAssignment; @@ -27,6 +28,8 @@ import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.commons.lang3.StringUtils; + /** * * @author qqmyers @@ -96,6 +99,24 @@ public Dataset execute(CommandContext ctxt) throws CommandException { updateVersion.getWorkflowComments().addAll(newComments); } + // Transfer curation status entries from draft to published version + List draftCurationStatuses = newVersion.getCurationStatuses(); + if (draftCurationStatuses != null && !draftCurationStatuses.isEmpty()) { + for (CurationStatus cs : draftCurationStatuses) { + // Update the dataset version reference + //This call sets the version in the curationstatus object as well + updateVersion.addCurationStatus(cs); + } + // Clear the list from the draft version + newVersion.getCurationStatuses().clear(); + } + + // Add a new empty curation status to indicate the curation action + CurationStatus status = updateVersion.getCurrentCurationStatus(); + if (status != null && StringUtils.isNotBlank(status.getLabel())) { + updateVersion.addCurationStatus(new CurationStatus(null, updateVersion, getRequest().getAuthenticatedUser())); + } + // we have to merge to update the database but not flush because // we don't want to create two draft versions! Dataset tempDataset = getDataset(); From 2837168183953505b8eb66a9d90e1374e0d1007f Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 19 Aug 2025 17:37:24 -0400 Subject: [PATCH 139/634] comment edit --- .../command/impl/CuratePublishedDatasetVersionCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index 21139ac3fbc..bf2fe78e721 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -111,7 +111,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { newVersion.getCurationStatuses().clear(); } - // Add a new empty curation status to indicate the curation action + // Add a new empty curation status to clear the status in the published version (as done in the FinalizeDatasetPublicationCommand) CurationStatus status = updateVersion.getCurrentCurationStatus(); if (status != null && StringUtils.isNotBlank(status.getLabel())) { updateVersion.addCurationStatus(new CurationStatus(null, updateVersion, getRequest().getAuthenticatedUser())); From 4477025b996056659b841b59a1e36581097c0784 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 19 Aug 2025 17:37:40 -0400 Subject: [PATCH 140/634] only show change curation status entry for draft version --- src/main/webapp/dataset.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index f1f72b50634..e9112997149 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -525,7 +525,7 @@ #{bundle['dataset.curationStatusMenu']}
    - +
  • #{bundle['dataset.changestatus']}
      From 56dc0a9f8381ed266785543fed4185f78fe73fff Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Aug 2025 15:41:32 -0400 Subject: [PATCH 141/634] random comment edits --- .../command/impl/CuratePublishedDatasetVersionCommand.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index bf2fe78e721..9ebdd28e009 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -76,7 +76,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { newTerms.setDatasetVersion(updateVersion); updateVersion.setTermsOfUseAndAccess(newTerms); - //Creation Note + //Version Note updateVersion.setVersionNote(newVersion.getVersionNote()); // Clear unnecessary terms relationships .... @@ -222,7 +222,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // This can be corrected by running the update PID API later, but who will look in the log? // With the change to not use the DeleteDatasetVersionCommand above and other // fixes, this error may now cleanly restore the initial state - // with the draft and last published versions unchanged, but this has not yet bee tested. + // with the draft and last published versions unchanged, but this has not yet been tested. // (Alternately this could move to onSuccess if we intend it to stay non-fatal.) logger.log(Level.WARNING, "Curate Published DatasetVersion: exception while updating PID metadata:{0}", ex.getMessage()); } From 0d01bbfc407ba7e6d3d3713fa3d628ca0823d359 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Aug 2025 16:05:51 -0400 Subject: [PATCH 142/634] add test of publish and update-current behavior --- .../harvard/iq/dataverse/api/DatasetsIT.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index b273502e6ed..4350a55420b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -3944,6 +3944,57 @@ public void testCurationStatusAPIs() { Response setInvalidStatus = UtilIT.setDatasetCurationLabel(datasetId, apiToken, "Invalid Status"); setInvalidStatus.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); + // Publish the dataset + UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken).then().assertThat() + .statusCode(OK.getStatusCode()); + + // Verify that the current curation label is now empty after publishing + Response getStatusAfterPublish = UtilIT.getDatasetCurationStatus(datasetId, apiToken, false); + getStatusAfterPublish.then().assertThat().statusCode(OK.getStatusCode()); + JsonObject statusAfterPublish = Json.createReader(new StringReader(getStatusAfterPublish.body().asString())).readObject(); + JsonObject dataObject = statusAfterPublish.getJsonObject("data"); + assertFalse(dataObject.containsKey("label"), "Curation label should be empty after publishing"); + + // Add a new valid curation label + Response setNewStatus = UtilIT.setDatasetCurationLabel(datasetId, apiToken, "State 2"); + setNewStatus.then().assertThat().statusCode(OK.getStatusCode()); + + // Verify the label was set + Response getStatusAfterSet = UtilIT.getDatasetCurationStatus(datasetId, apiToken, false); + getStatusAfterSet.then().assertThat().statusCode(OK.getStatusCode()); + JsonObject statusAfterSet = Json.createReader(new StringReader(getStatusAfterSet.body().asString())).readObject(); + assertEquals("State 2", statusAfterSet.getString("label"), "Curation label should be set to State 2"); + + // Publish the dataset again using updatecurrent as superuser + Response updateCurrentResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "updatecurrent", apiToken); + updateCurrentResponse.then().assertThat().statusCode(OK.getStatusCode()); + + // Verify that the current curation label is now empty after updatecurrent + Response getStatusAfterUpdateCurrent = UtilIT.getDatasetCurationStatus(datasetId, apiToken, false); + getStatusAfterUpdateCurrent.then().assertThat().statusCode(OK.getStatusCode()); + JsonObject statusAfterUpdateCurrent = Json.createReader(new StringReader(getStatusAfterUpdateCurrent.body().asString())).readObject(); + JsonObject dataAfterUpdateCurrent = statusAfterUpdateCurrent.getJsonObject("data"); + assertFalse(dataAfterUpdateCurrent.containsKey("label"), "Curation label should be empty after updatecurrent"); + + // Verify that the history contains the previously added label + Response getHistoryAfterUpdateCurrent = UtilIT.getDatasetCurationStatus(datasetId, apiToken, true); + getHistoryAfterUpdateCurrent.then().assertThat().statusCode(OK.getStatusCode()); + + // Extract the data array from the response + JsonObject responseObj = Json.createReader(new StringReader(getHistoryAfterUpdateCurrent.body().asString())).readObject(); + JsonArray historyAfterUpdateCurrent = responseObj.getJsonArray("data"); + + // Verify history contains the State 2 label + boolean foundState2 = false; + for (int i = 0; i < historyAfterUpdateCurrent.size(); i++) { + JsonObject entry = historyAfterUpdateCurrent.getJsonObject(i); + if (entry.containsKey("label") && "State 2".equals(entry.getString("label"))) { + foundState2 = true; + break; + } + } + assertTrue(foundState2, "History should contain the State 2 label after updatecurrent"); + // Clean up Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); assertEquals(200, deleteDatasetResponse.getStatusCode()); From 0fc4ba3390409e55e7b6c34bf88cae89fc949c67 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 29 Aug 2025 16:15:24 -0400 Subject: [PATCH 143/634] release note --- doc/release-notes/11783-Curation-Status-fixes.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/release-notes/11783-Curation-Status-fixes.md diff --git a/doc/release-notes/11783-Curation-Status-fixes.md b/doc/release-notes/11783-Curation-Status-fixes.md new file mode 100644 index 00000000000..62b33165d0c --- /dev/null +++ b/doc/release-notes/11783-Curation-Status-fixes.md @@ -0,0 +1 @@ +In prior versions of Dataverse, publishing a dataset via the superuser only update-current-version option would not set the current curation status (if enabled/used) to none/empty and, in v6.7, would not maintain the curation status history. These issues are now resolved and the update current version option works the same as normal publication of a new version w.r.t. curation status. \ No newline at end of file From fdf75cd7a0ed96195a205badb31a6e7ed84233d1 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 2 Sep 2025 10:13:57 -0400 Subject: [PATCH 144/634] removing some experimental lines from the Exporter interface #11766 --- .../src/main/java/io/gdcc/spi/export/Exporter.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/Exporter.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/Exporter.java index f91a2504f74..7132e74641b 100644 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/Exporter.java +++ b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/Exporter.java @@ -84,19 +84,6 @@ public interface Exporter { default Optional getPrerequisiteFormatName() { return Optional.empty(); } - - /** - * Most metadata formats do not require tabular metadata to be generated. - * (the 2 known cases that encode variable-level information, as of Dataverse 6.7, - * are rich DDI and Croissant). In order to serve it more efficiently, - * the data provider may need to be initialized in a special way (with access - * to EJBs that in most cases isn't needed), so it may help to know - * programmatically whether it's required for this format. - * @return Optional - */ - default Optional isNeedsTabularMetadata() { - return Optional.of(false); - } /** * Harvestable Exporters will be available as options in Dataverse's Harvesting mechanism. From 4ca3f3348023939c0d35a275fa770eb36fc97d16 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 2 Sep 2025 10:20:33 -0400 Subject: [PATCH 145/634] change defaultToOwner to getEffective --- doc/release-notes/11695-change-api-get-storage-driver.md | 2 +- doc/sphinx-guides/source/admin/dataverses-datasets.rst | 2 +- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 4 ++-- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/release-notes/11695-change-api-get-storage-driver.md b/doc/release-notes/11695-change-api-get-storage-driver.md index cd78c642a2a..aa993232f4d 100644 --- a/doc/release-notes/11695-change-api-get-storage-driver.md +++ b/doc/release-notes/11695-change-api-get-storage-driver.md @@ -7,6 +7,6 @@ The API for getting the Storage Driver info has been changed/extended. /api/admin/dataverse/{dataverse-alias}/storageDriver changed "message" to "name" and added "type" and "label" -Also added query param for /api/admin/dataverse/{dataverse-alias}/storageDriver?defaultToOwner=true to recurse the chain of parents to find the effective storageDriver +Also added query param for /api/admin/dataverse/{dataverse-alias}/storageDriver?getEffective=true to recurse the chain of parents to find the effective storageDriver See also [the guides](https://dataverse-guide--11664.org.readthedocs.build/en/11664/api/native-api.html#configure-a-dataset-to-store-all-new-files-in-a-specific-file-store), #11695, and #11664. diff --git a/doc/sphinx-guides/source/admin/dataverses-datasets.rst b/doc/sphinx-guides/source/admin/dataverses-datasets.rst index fcfbd0c62f3..0fa5bcf69f1 100644 --- a/doc/sphinx-guides/source/admin/dataverses-datasets.rst +++ b/doc/sphinx-guides/source/admin/dataverses-datasets.rst @@ -62,7 +62,7 @@ The current driver can be seen using:: Or to recurse the chain of parents to find the effective storageDriver:: - curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver?defaultToOwner=true + curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver?getEffective=true (Note that for ``dataverse.files.store1.label=MyLabel``, ``store1`` will be returned.) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 48246820bc7..003698a2981 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -2184,7 +2184,7 @@ public Response addRoleAssignementsToChildren(@Context ContainerRequestContext c @AuthRequired @Path("/dataverse/{alias}/storageDriver") public Response getStorageDriver(@Context ContainerRequestContext crc, @PathParam("alias") String alias, - @QueryParam("defaultToOwner") Boolean defaultToOwner) throws WrappedResponse { + @QueryParam("getEffective") Boolean getEffective) throws WrappedResponse { Dataverse dataverse = dataverseSvc.findByAlias(alias); if (dataverse == null) { return error(Response.Status.NOT_FOUND, "Could not find dataverse based on alias supplied: " + alias + "."); @@ -2198,7 +2198,7 @@ public Response getStorageDriver(@Context ContainerRequestContext crc, @PathPara return wr.getResponse(); } - if (defaultToOwner != null && defaultToOwner) { + if (getEffective != null && getEffective) { return ok(JsonPrinter.jsonStorageDriver(dataverse.getEffectiveStorageDriverId(), null)); } else { return ok(JsonPrinter.jsonStorageDriver(dataverse.getStorageDriverId(), null)); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 67028d8be06..3949bcd42fd 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2913,8 +2913,8 @@ static Response listStorageDrivers(String apiToken) { static Response getStorageDriver(String dvAlias, String apiToken) { return getStorageDriver(dvAlias, apiToken, null); } - static Response getStorageDriver(String dvAlias, String apiToken, Boolean defaultToOwner) { - String params = defaultToOwner != null ? "?defaultToOwner=" + defaultToOwner : ""; + static Response getStorageDriver(String dvAlias, String apiToken, Boolean getEffective) { + String params = getEffective != null ? "?getEffective=" + getEffective : ""; return given() .header(API_TOKEN_HTTP_HEADER, apiToken) .get("/api/admin/dataverse/" + dvAlias + "/storageDriver" + params); From 2aaef06225a3d807cc78b65691bfbf2086090c14 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 2 Sep 2025 11:43:42 -0400 Subject: [PATCH 146/634] fix test --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 4350a55420b..f7f42b4b4a6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -3875,6 +3875,8 @@ public void testCurationStatusAPIs() { createDataverseResponse.prettyPrint(); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken).then().assertThat().statusCode(OK.getStatusCode()); + Response setCurationLabelSets = UtilIT.setSetting(SettingsServiceBean.Key.AllowedCurationLabels, "{\"StandardProcess\":[\"Author contacted\", \"Privacy Review\", \"Awaiting paper publication\", \"Final Approval\"],\"AlternateProcess\":[\"State 1\",\"State 2\",\"State 3\"]}"); setCurationLabelSets.then().assertThat() .statusCode(OK.getStatusCode()); @@ -3885,7 +3887,7 @@ public void testCurationStatusAPIs() { Response setDataverseCurationLabelSetResponse = UtilIT.setDataverseCurationLabelSet(dataverseAlias, apiToken, "AlternateProcess"); setDataverseCurationLabelSetResponse.then().assertThat().statusCode(FORBIDDEN.getStatusCode()); - Response makeSuperUser = UtilIT.makeSuperUser(username); + Response makeSuperUser = UtilIT.setSuperuserStatus(username, true); assertEquals(200, makeSuperUser.getStatusCode()); //Non-existent option @@ -3996,7 +3998,7 @@ public void testCurationStatusAPIs() { assertTrue(foundState2, "History should contain the State 2 label after updatecurrent"); // Clean up - Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); + Response deleteDatasetResponse = UtilIT.destroyDataset(datasetId, apiToken); assertEquals(200, deleteDatasetResponse.getStatusCode()); Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); From 862522eb7cbcc6d70a60d58570760974936fd39d Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 2 Sep 2025 12:46:32 -0400 Subject: [PATCH 147/634] a release note for the dataverse-spi update. #11766 --- doc/release-notes/11766-new-io.gdcc.dataverse-spi.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 doc/release-notes/11766-new-io.gdcc.dataverse-spi.md diff --git a/doc/release-notes/11766-new-io.gdcc.dataverse-spi.md b/doc/release-notes/11766-new-io.gdcc.dataverse-spi.md new file mode 100644 index 00000000000..ef9e68f5719 --- /dev/null +++ b/doc/release-notes/11766-new-io.gdcc.dataverse-spi.md @@ -0,0 +1,2 @@ +The ExportDataProvider framework in the dataverse-spi package has been extended, adding some extra options for developers of metadata exporter plugins. +See the [documentation](https://guides.dataverse.org/en/latest/developers/metadataexport.html#building-an-exporter) in the Metadata Export guide for details. \ No newline at end of file From 7a8d8f850a720d6abcddbf4f91f5f09b88e0edb8 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 2 Sep 2025 13:42:26 -0400 Subject: [PATCH 148/634] viewing the status/history with no draft is ok --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 729174dedfc..8628a45574d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -2609,7 +2609,7 @@ public Response getCurationStatus(@Context ContainerRequestContext crc, canSeeStatus = permissionSvc.requestOn(createDataverseRequest(user), ds).has(Permission.PublishDataset); } - if (dsv.isDraft() && (canSeeStatus)) { + if (canSeeStatus) { List statuses = includeHistory ? dsv.getCurationStatuses() : Collections.singletonList(dsv.getCurrentCurationStatus()); if (includeHistory) { JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); From e902ed4a4e5ff8eb96967160b6d38f4139cf5e5f Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 3 Sep 2025 06:57:26 -0400 Subject: [PATCH 149/634] cause new draft --- .../java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index f7f42b4b4a6..755de140db8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -3956,7 +3956,13 @@ public void testCurationStatusAPIs() { JsonObject statusAfterPublish = Json.createReader(new StringReader(getStatusAfterPublish.body().asString())).readObject(); JsonObject dataObject = statusAfterPublish.getJsonObject("data"); assertFalse(dataObject.containsKey("label"), "Curation label should be empty after publishing"); - + + //Cause a new draft version + String jsonLDTerms = "{\"https://dataverse.org/schema/core#fileTermsOfAccess\":{\"https://dataverse.org/schema/core#dataAccessPlace\":\"Somewhere\"}}"; + Response updateTerms = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, jsonLDTerms, true); + updateTerms.then().assertThat() + .statusCode(OK.getStatusCode()); + // Add a new valid curation label Response setNewStatus = UtilIT.setDatasetCurationLabel(datasetId, apiToken, "State 2"); setNewStatus.then().assertThat().statusCode(OK.getStatusCode()); From e63ac43ab62a435de5447b947e843bcc8e72b18c Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 3 Sep 2025 10:06:29 -0400 Subject: [PATCH 150/634] missed getting data element --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 755de140db8..5dac4b1d05c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -3971,7 +3971,8 @@ public void testCurationStatusAPIs() { Response getStatusAfterSet = UtilIT.getDatasetCurationStatus(datasetId, apiToken, false); getStatusAfterSet.then().assertThat().statusCode(OK.getStatusCode()); JsonObject statusAfterSet = Json.createReader(new StringReader(getStatusAfterSet.body().asString())).readObject(); - assertEquals("State 2", statusAfterSet.getString("label"), "Curation label should be set to State 2"); + JsonObject dataInSecondDraft = statusAfterSet.getJsonObject("data"); + assertEquals("State 2", dataInSecondDraft.getString("label"), "Curation label should be set to State 2"); // Publish the dataset again using updatecurrent as superuser Response updateCurrentResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "updatecurrent", apiToken); From 1d5199bc79738ad7c3bf41d959c7b5957f2718c3 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 3 Sep 2025 10:06:59 -0400 Subject: [PATCH 151/634] Update native-api.rst --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 04aaf59de98..f6b2cfa866b 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1106,7 +1106,7 @@ Update Collection Input Levels Updates the dataset field type input levels in a collection. -Please note that this endpoint overwrites all the input levels of the collection page, so if you want to keep the existing ones, you will need to add them to the JSON request body. +Please note that this endpoint does not change previously updated input levels of the collection page, so if you want to keep the modify exisitng ones, you will need to include them in the JSON request body. If one of the input levels corresponds to a dataset field type belonging to a metadata block that does not exist in the collection, the metadata block will be added to the collection. From 35f2dd40c650731cd5715cbba3768542706935e2 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 3 Sep 2025 10:07:49 -0400 Subject: [PATCH 152/634] #11387 update tests from code review --- .../java/edu/harvard/iq/dataverse/api/DataversesIT.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 682b2ddc8a3..f6b091f41f5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1264,8 +1264,8 @@ public void testUpdateInputLevels() { boolean [] testDisplayOnCreate = new boolean[] {true, false}; updateDataverseInputLevelsResponse = UtilIT.updateDataverseInputLevels(dataverseAlias, testInputLevelNames, testRequiredInputLevels, testIncludedInputLevels, testDisplayOnCreate, apiToken); updateDataverseInputLevelsResponse.prettyPrint(); - int subtitleInputLevelIndex = 0; - int relatedMaterialInputLevelIndex = 0; + int subtitleInputLevelIndex = -1; + int relatedMaterialInputLevelIndex = -1; int i = 0; while (updateDataverseInputLevelsResponse.then().extract().path(String.format("data.inputLevels[%d].datasetFieldTypeName", i)) != null){ @@ -1304,7 +1304,6 @@ public void testUpdateInputLevels() { updateDataverseInputLevelsResponse.prettyPrint(); subtitleInputLevelIndex = 0; - int otherReferencesInputLevelIndex = 0; i = 0; while (updateDataverseInputLevelsResponse.then().extract().path(String.format("data.inputLevels[%d].datasetFieldTypeName", i)) != null) { @@ -1312,9 +1311,6 @@ public void testUpdateInputLevels() { if (actualInputLevelName.equals("subtitle")) { subtitleInputLevelIndex = i; } - if (actualInputLevelName.equals("otherReferences")) { - otherReferencesInputLevelIndex = i; - } i++; } From a966923eb0ac27ea999fbf9138ff7746036aad83 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 3 Sep 2025 10:17:50 -0400 Subject: [PATCH 153/634] #11387 add release note. --- doc/release-notes/11387-modify-input-level-api.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/release-notes/11387-modify-input-level-api.md diff --git a/doc/release-notes/11387-modify-input-level-api.md b/doc/release-notes/11387-modify-input-level-api.md new file mode 100644 index 00000000000..ca18eabb515 --- /dev/null +++ b/doc/release-notes/11387-modify-input-level-api.md @@ -0,0 +1,3 @@ +### Update Collection Input Level API Changed + +- This endpoint will no longer delete the custom input levels previously modified for the given collection. In order to update a previously modified custom input level, it must be included in the JSON provided to the api. From 6db78587f94d04da77b9c8ede37069d978b072f8 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 3 Sep 2025 15:48:07 -0400 Subject: [PATCH 154/634] improve "making release" doc post 6.7/6.7.1 #11790 --- .../source/developers/making-releases.rst | 80 ++++++++++++++++--- .../source/qa/testing-approach.md | 2 +- 2 files changed, 72 insertions(+), 10 deletions(-) diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index 028b80e2892..4c9fd28b678 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -8,9 +8,13 @@ Making Releases Introduction ------------ -This document is about releasing the main Dataverse app (https://github.com/IQSS/dataverse). See :doc:`making-library-releases` for how to release our various libraries. Other projects have their own release documentation. +.. note:: This document is about releasing the main Dataverse app (https://github.com/IQSS/dataverse). See :doc:`making-library-releases` for how to release our various libraries. Other projects have their own release documentation. -Below you'll see branches like "develop" and "master" mentioned. For more on our branching strategy, see :doc:`version-control`. +.. note:: Below you'll see branches like "develop" and "master" mentioned. For more on our branching strategy, see :doc:`version-control`. + +Dataverse releases are time-based as opposed to being feature-based. That is, we announce an approximate release date in advance (e.g. for `6.8 `_) and try to hit that deadline. If features we're working on aren't ready yet, the train will leave the station without them. We release quarterly. + +We also announce "last call" dates for both community pull requests and those made by core developers. If you are part of the community and have made a pull request, you have until this date to ask the team to add the upcoming milestone to your pull request. The same goes for core developers. This is not a guarantee that these pull requests will be reviewed, tested, QA'ed and merged before :ref:`code freeze `, but we'll try. Regular or Hotfix? ------------------ @@ -30,7 +34,9 @@ Early on, make sure it's clear what type of release this is. The steps below des Ensure Issues Have Been Created ------------------------------- -Some of the steps in this document are well-served by having their own dedicated GitHub issue. You'll see a label like this on them: +We have a "create release issues" script at https://github.com/IQSS/sabo that should be run a week or so before code freeze. + +For each issue that is created by the script there is likely a corresponding step in this document that has "dedicated" label on it like this: |dedicated| @@ -41,20 +47,39 @@ There are a variety of reasons why a step might deserve its own dedicated issue: Steps don't get their own dedicated issue if it would be confusing to have multiple people involved. Too many cooks in the kitchen, as they say. Also, some steps are so small the overhead of an issue isn't worth it. -Before the release even begins you can coordinate with the project manager about the creation of these issues. - .. |dedicated| raw:: html Dedicated Issue   +.. _declare-code-freeze: + Declare a Code Freeze --------------------- -The following steps are made more difficult if code is changing in the "develop" branch. Declare a code freeze until the release is out. Do not allow pull requests to be merged. +When we declare a code freeze, we mean: + +- No additional features will be merged until the freeze is lifted. +- Bug fixes will only be merged if they relate to the upcoming release in some way, such as fixes for regressions or performance problems in that release. +- Pull requests that directly affect the release, such as bumping the version, will be merged, of course. + +The benefits of the code freeze are: + +- The team can focus on getting the release out together. +- Regression and performance testing can happen on code that isn't changing. +- The release notes can be written without having to worry about new features (and their release note snippets) being merged in. + +In short, the steps described below become easier under a code freeze. -For a hotfix, a code freeze (no merging) is necessary not because we want code to stop changing in the branch being hotfix released, but because bumping the version used in Jenkins/Ansible means that API tests will fail in pull requests until the version is bumped in those pull requests. +Note: for a hotfix, a code freeze is necessary not because we want code to stop changing in the branch being hotfix released, but because bumping the version used in Jenkins/Ansible means that API tests will fail in pull requests until the version is bumped in those pull requests. Basically, we want to get the hotfix merged quickly so we can propogate the version bump into all open pull requests so that API tests can start passing again in those pull requests. + +Push Back Milestones on Pull Requests That Missed the Train +----------------------------------------------------------- + +As of this writing, we optimistically add milestones to issues and pull requests, hoping that the work will be complete before code freeze. Inevitably, we're a bit too optimistic. + +Hopefully, as the release approached, the team has already decided which pull requests (that aren't related to the release) won't make the cut. If not, go ahead and bump them to the next release. Conduct Performance Testing --------------------------- @@ -68,6 +93,7 @@ Conduct Regression Testing |dedicated| +Regression testing should be conducted on production data. See :doc:`/qa/testing-approach` for details. Refer to the provided regression checklist for the list of items to verify during the testing process: `Regression Checklist `_. @@ -85,7 +111,7 @@ The task at or near release time is to collect these snippets into a single file - Find the issue in GitHub that tracks the work of creating release notes for the upcoming release. - Create a branch, add a .md file for the release (ex. 5.10.1 Release Notes) in ``/doc/release-notes`` and write the release notes, making sure to pull content from the release note snippets mentioned above. Snippets may not include any issue number or pull request number in the text so be sure to copy the number from the filename of the snippet into the final release note. - Delete (``git rm``) the release note snippets as the content is added to the main release notes file. -- Include instructions describing the steps required to upgrade the application from the previous version. These must be customized for release numbers and special circumstances such as changes to metadata blocks and infrastructure. +- Include instructions describing the steps required to upgrade the application from the previous version. These must be customized for release numbers and special circumstances such as changes to metadata blocks and infrastructure. These instructions are required for the next steps (deploying to various environments) so try to prioritize them over finding just the right words in release highlights (which you can do later). - Make a pull request. Here's an example: https://github.com/IQSS/dataverse/pull/11613 - Note that we won't merge the release notes until after we have confirmed that the upgrade instructions are valid by performing a couple upgrades. @@ -110,6 +136,13 @@ ssh into the dataverse-internal server and download the release candidate war fi Go to /doc/release-notes, open the release-notes.md file for the release we're working on, and perform all the steps under "Upgrade Instructions". Note that for regular releases, we haven't bumped the version yet so you won't be able to follow the steps exactly. (For hotfix releases, the version will be bumped already.) +Deploy Release Candidate to QA +------------------------------ + +|dedicated| + +Deploy the same war file to https://qa.dataverse.org using the same upgrade instructions as above. + Deploy Release Candidate to Demo -------------------------------- @@ -117,6 +150,26 @@ Deploy Release Candidate to Demo Deploy the same war file to https://demo.dataverse.org using the same upgrade instructions as above. +Solicit Feedback from Curation Team +----------------------------------- + +Ask the curation team to test on https://qa.dataverse.org and give them five days to provide feedback. + +Build the Guides for the Release Candidate +------------------------------------------ + +Go to https://jenkins.dataverse.org/job/guides.dataverse.org/ and make the following adjustments to the config: + +- Repository URL: ``https://github.com/IQSS/dataverse.git`` +- Branch Specifier (blank for 'any'): ``*/develop`` +- ``VERSION`` (under "Build Steps"): use the next release version but add "-rc.1" to the end. Don't prepend a "v". Use ``6.8-rc.1`` (for example) + +Click "Save" then "Build Now". + +Make sure the guides directory appears in the expected location such as https://guides.dataverse.org/en/6.8-rc.1/ + +When previewing the HTML version of docs from pull requests, we don't usually use this Jenkins job, relying instead on automated ReadTheDocs builds. The reason for doing this step now while we wait for feedback from the Curation Team is that it's an excellent time to fix the Jenkins job, if necessary, to accomodate any changes needed to continue to build the docs. For example, Sphinx might need to be updated or a dependency might need to be installed. Such changes should be listed in the release notes for documentation writers. + Merge Release Notes (Once Ready) -------------------------------- @@ -171,10 +224,19 @@ Merge "develop" into "master" (non-hotfix only) If this is a regular (non-hotfix) release, create a pull request to merge the "develop" branch into the "master" branch using this "compare" link: https://github.com/IQSS/dataverse/compare/master...develop -Once important tests have passed (compile, unit tests, etc.), merge the pull request (skipping code review is ok). Don't worry about style tests failing such as for shell scripts. +Allow time for important tests (compile, unit tests, etc.) to pass. Don't worry about style tests failing such as for shell scripts. It's ok to skip code review. + +When merging the pull request, be sure to choose "create a merge commit" and not "squash and merge" or "rebase and merge". We suspect that choosing squash or rebase may have led to `lots of merge conflicts `_ when we tried to perform this "merge develop to master" step, forcing us to `re-do `_ the previous release before we could proceed with the current release. If this is a hotfix release, skip this whole "merge develop to master" step (the "develop" branch is not involved until later). +Confirm "master" Mergeability +----------------------------- + +Hopefully, the previous step went ok. As a sanity check, use the "compare" link at https://github.com/IQSS/dataverse/compare/master...develop again to look for merge conflicts without making a pull request. + +If the GitHub UI tells you there would be merge conflicts, something has gone horribly wrong (again) with the "merge develop to master" step. Stop and ask for help. + Add Milestone to Pull Requests and Issues ----------------------------------------- diff --git a/doc/sphinx-guides/source/qa/testing-approach.md b/doc/sphinx-guides/source/qa/testing-approach.md index 817161d02a0..49b9075cf7f 100644 --- a/doc/sphinx-guides/source/qa/testing-approach.md +++ b/doc/sphinx-guides/source/qa/testing-approach.md @@ -34,7 +34,7 @@ Think about risk. Is the feature or function part of a critical area such as per ## Smoke Test -1. Go to the homepage on . Scroll to the bottom to ensure the build number is the one you intend to test from Jenkins. +1. Go to the homepage on (this server has production data). Scroll to the bottom to ensure the build number is the one you intend to test from Jenkins. 1. Create a new user: It's fine to use a formulaic name with your initials and date and make the username and password the same, eg. kc080622. 1. Create a dataverse: You can use the same username. 1. Create a dataset: You can use the same username; fill in the required fields (do not use a template). From 5b4eab384126e370f4af5c1bab38609c5bf69705 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 4 Sep 2025 14:13:28 -0400 Subject: [PATCH 155/634] Update changelog.rst --- doc/sphinx-guides/source/api/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 16157459220..3de91d8d3b2 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -13,6 +13,7 @@ v6.8 - For POST /api/files/{id}/metadata passing an empty string ("description":"") or array ("categories":[]) will no longer be ignored. Empty fields will now clear out the values in the file's metadata. To ignore the fields simply do not include them in the JSON string. - For PUT /api/datasets/{id}/editMetadata the query parameter "sourceInternalVersionNumber" has been removed and replaced with "sourceLastUpdateTime" to verify that the data being edited hasn't been modified and isn't stale. - For GET /api/dataverses/$dataverse-alias/links the Json response has changed breaking the backward compatibility of the API. +- For PUT /api/dataverses/$dataverse-alias/inputLevels custom input levels that had been previously set will no longer be deleted. v6.7 ---- From f8757b4c34b43522faa91e33325e3cd3aa7a048f Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 4 Sep 2025 16:24:23 -0400 Subject: [PATCH 156/634] fix termName parsing --- .../dataverse/pidproviders/doi/XmlMetadataTemplate.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 1bd0f4a1adc..9e13728bc73 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -58,6 +58,8 @@ import edu.harvard.iq.dataverse.util.xml.XmlWriterUtil; import jakarta.enterprise.inject.spi.CDI; import jakarta.json.JsonObject; +import jakarta.json.JsonString; +import jakarta.json.JsonValue; import jakarta.json.JsonValue.ValueType; public class XmlMetadataTemplate { @@ -623,8 +625,11 @@ private void writeEntityElements(XMLStreamWriter xmlw, String elementName, Strin isROR = true; JsonObject jo = getExternalVocabularyValue(orgName); // Some ext. cvv configs store a JsonArray of multiple objects/values. In such cases, we'll leave orgName blank - if (jo != null && jo.getValueType() == ValueType.STRING) { - orgName = jo.getString("termName"); + if (jo.containsKey("termName")) { + JsonValue termName = jo.get("termName"); + if (termName.getValueType() == ValueType.STRING) { + orgName = ((JsonString) termName).getString(); + } } } From f7157ef555e56eb475c3168d135f3767fcfa8625 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 4 Sep 2025 17:56:15 -0400 Subject: [PATCH 157/634] add null check --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 9e13728bc73..9ebb346baf8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -625,7 +625,7 @@ private void writeEntityElements(XMLStreamWriter xmlw, String elementName, Strin isROR = true; JsonObject jo = getExternalVocabularyValue(orgName); // Some ext. cvv configs store a JsonArray of multiple objects/values. In such cases, we'll leave orgName blank - if (jo.containsKey("termName")) { + if (jo != null && jo.containsKey("termName")) { JsonValue termName = jo.get("termName"); if (termName.getValueType() == ValueType.STRING) { orgName = ((JsonString) termName).getString(); From 97e3803388f21bad388f023e8f716d9b79bca5c3 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 4 Sep 2025 18:57:58 -0400 Subject: [PATCH 158/634] switch from ossrh to central #11766 #11512 --- .github/workflows/spi_release.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/spi_release.yml b/.github/workflows/spi_release.yml index 1d29cdf76c9..098e094902d 100644 --- a/.github/workflows/spi_release.yml +++ b/.github/workflows/spi_release.yml @@ -42,8 +42,7 @@ jobs: with: java-version: '17' distribution: 'adopt' - # TODO: change this from ossrh to central? - server-id: ossrh + server-id: central server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD - uses: actions/cache@v4 @@ -81,8 +80,7 @@ jobs: with: java-version: '17' distribution: 'adopt' - # TODO: change this from ossrh to central? - server-id: ossrh + server-id: central server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD gpg-private-key: ${{ secrets.DATAVERSEBOT_GPG_KEY }} From b52a791bf227091d2caf890ce9e16aa20cd1034f Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 5 Sep 2025 09:01:24 -0400 Subject: [PATCH 159/634] adding v5 to action.yml --- .github/actions/setup-maven/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-maven/action.yml b/.github/actions/setup-maven/action.yml index 4cf09f34231..c3bc94b0d28 100644 --- a/.github/actions/setup-maven/action.yml +++ b/.github/actions/setup-maven/action.yml @@ -23,7 +23,7 @@ runs: echo "JAVA_VERSION=$(grep '' ${GITHUB_WORKSPACE}/modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" | tee -a ${GITHUB_ENV} - name: Set up JDK ${{ env.JAVA_VERSION }} id: setup-java - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: ${{ env.JAVA_VERSION }} distribution: 'temurin' From c3a13c893284b4376ee8db5c39364da4492068bb Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 4 Sep 2025 14:14:26 -0400 Subject: [PATCH 160/634] add support for matching object with a key in an array of strings --- .../iq/dataverse/DatasetFieldServiceBean.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index 0b6b74e6a73..811c67c52d6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -764,13 +764,24 @@ Object processPathSegment(int index, String[] pathParts, JsonValue curPath, Stri JsonValue val = jo.get(keyVal[0]); if (val != null) { if (val.getValueType().equals(ValueType.STRING)) { + //Match a string value if (((JsonString) val).getString().equals(expected)) { logger.fine("Found: " + jo); curPath = jo; return processPathSegment(index + 1, pathParts, curPath, termUri); } - } else { - logger.warning("Expected a string value for " + keyVal[0] + " but found: " + val.getValueType()); + } else if (val.getValueType() == JsonValue.ValueType.ARRAY) { + // Match one string in an array + JsonArray jsonArray = (JsonArray) val; + for (JsonValue arrayVal : jsonArray) { + if (arrayVal.getValueType() == JsonValue.ValueType.STRING) { + if (((JsonString) arrayVal).getString().equals(expected)) { + logger.fine("Found match in array: " + jo.toString()); + curPath = jo; + return processPathSegment(index + 1, pathParts, curPath, termUri); + } + } + } } } } From 5ed86a1cde6f20d6b4fd8b6a2d9a232b66ffea1e Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 5 Sep 2025 13:39:01 -0400 Subject: [PATCH 161/634] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 327e31124f6..6d6b25b10ef 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -732,7 +732,7 @@ Note: you must have "Add Dataset" permission in the given collection to invoke t .. _featured-collections: -List Dataverse Collections to which a given Dataset or Dataverse Collection may be linked +List Dataverse Collections to Which a Given Dataset or Dataverse Collection May Be Linked ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The user may provide a search term to limit the list of Dataverse Collections returned. The search term will be compared to the name of the Dataverse Collections. From a3065927348fd44a637bd0c85c370620e9e2f3ce Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 5 Sep 2025 13:39:52 -0400 Subject: [PATCH 162/634] Update src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java Co-authored-by: Guillermo Portas --- .../java/edu/harvard/iq/dataverse/PermissionServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index a4111963ea2..303bc87d116 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -101,7 +101,7 @@ WITH grouplist AS ( WHERE explicitgroup_authenticateduser.containedauthenticatedusers_id = @USERID ) - SELECT * FROM DATAVERSE WHERE id IN ( + SELECT * FROM DATAVERSE WHERE id IN ( SELECT definitionpoint_id FROM roleassignment WHERE roleassignment.assigneeidentifier IN ( From 6cc2f6b05eb8835702bb0d9b1ca1776a0bb658d4 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 5 Sep 2025 13:40:45 -0400 Subject: [PATCH 163/634] Update src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java Co-authored-by: Guillermo Portas --- .../java/edu/harvard/iq/dataverse/PermissionServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index 303bc87d116..7deae81160a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -157,7 +157,7 @@ AND EXISTS (SELECT id FROM dataverserole WHERE dataverserole.id = roleassignment AND @IPRANGESQL ) ) - ) + ) """; /** * A request-level permission query (e.g includes IP ras). From f7f0d26ace4f0ebc7de0a6b7671452ec71614121 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 5 Sep 2025 13:41:05 -0400 Subject: [PATCH 164/634] Update src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java Co-authored-by: Guillermo Portas --- .../java/edu/harvard/iq/dataverse/PermissionServiceBean.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index 7deae81160a..742505acdf5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -961,7 +961,6 @@ public List findPermittedCollections(DataverseRequest request, Authen return null; } - /** * Calculates the complete list of role assignments for a given user on a DvObject. * This includes roles assigned directly to the user and roles inherited from any groups From f071e214345e4015fe64d880ae807f57ff633d4b Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 5 Sep 2025 13:41:30 -0400 Subject: [PATCH 165/634] Update src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java Co-authored-by: Guillermo Portas --- .../java/edu/harvard/iq/dataverse/PermissionServiceBean.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index 742505acdf5..7ae24a08c27 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -922,7 +922,6 @@ public List findPermittedCollections(DataverseRequest request, Authen return findPermittedCollections(request, user, 1 << permission.ordinal()); } - public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit) { if (user != null) { // IP Group - Only check IP if a User is calling for themself From fc68f5677646c0f6384595bea14e51ebc44680de Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 5 Sep 2025 13:55:24 -0400 Subject: [PATCH 166/634] release note --- doc/release-notes/11793 - ext. vocab. improvements.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/release-notes/11793 - ext. vocab. improvements.md diff --git a/doc/release-notes/11793 - ext. vocab. improvements.md b/doc/release-notes/11793 - ext. vocab. improvements.md new file mode 100644 index 00000000000..46998e56b60 --- /dev/null +++ b/doc/release-notes/11793 - ext. vocab. improvements.md @@ -0,0 +1 @@ +This version of Dataverse includes extensions of the Dataverse External Vocabulary mechanism (https://guides.dataverse.org/en/latest/admin/metadatacustomization.html#using-external-vocabulary-services) that improve Dataverse's ability to include metadata about vocabulary terms and external identifiers such as ORCID and ROR in it's metadata exports. More information on how to configure external vocabulary scripts to use this functionality can be found at https://github.com/gdcc/dataverse-external-vocab-support/blob/main/docs/readme.md and in the examples in the https://github.com/gdcc/dataverse-external-vocab-support repository. \ No newline at end of file From 5d8eabba0813d53c57fe3c627ef615b5b203e4ca Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 5 Sep 2025 14:01:30 -0400 Subject: [PATCH 167/634] Update src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java Co-authored-by: Guillermo Portas --- .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 18896df32ad..cc9d85d0ad8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -291,7 +291,7 @@ public static JsonObjectBuilder json(Workflow wf){ return bld; } - public static JsonObjectBuilder json(Dataverse dv, Boolean mini) { + public static JsonObjectBuilder json(Dataverse dv, boolean minimal) { if (!mini){ return json(dv, false, false, null); } else { From e975a68289e791ec28694e206128692255eacf72 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 5 Sep 2025 14:03:30 -0400 Subject: [PATCH 168/634] Update src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java Co-authored-by: Guillermo Portas --- src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 1e79706b7f7..8b0f5323659 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -797,7 +797,7 @@ public void testGetLinkableDataverses(){ .statusCode(OK.getStatusCode()) .body("data.size()", equalTo(0)); - //set user api back to super user for cleanup + //set user api back to super user for cleanup UtilIT.setSuperuserStatus(username, Boolean.TRUE); From 70b16d596821ded7bf5a4a12cdc6b2a7b81e1886 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 5 Sep 2025 14:42:25 -0400 Subject: [PATCH 169/634] #11710 fix typo --- .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index cc9d85d0ad8..779cb84e9cc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -292,7 +292,7 @@ public static JsonObjectBuilder json(Workflow wf){ } public static JsonObjectBuilder json(Dataverse dv, boolean minimal) { - if (!mini){ + if (!minimal){ return json(dv, false, false, null); } else { return jsonObjectBuilder() From e7c344846910f9db40399b273fcfeefee07f2f85 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 5 Sep 2025 15:48:39 -0400 Subject: [PATCH 170/634] show draft/unpublished in UI --- ...18-file-level-permissions-restricted-draft.md | 3 +++ .../java/edu/harvard/iq/dataverse/DataFile.java | 16 +++++++++++++++- src/main/java/propertyFiles/Bundle.properties | 1 + src/main/webapp/permissions-manage-files.xhtml | 1 + 4 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 doc/release-notes/7618-file-level-permissions-restricted-draft.md diff --git a/doc/release-notes/7618-file-level-permissions-restricted-draft.md b/doc/release-notes/7618-file-level-permissions-restricted-draft.md new file mode 100644 index 00000000000..6b674ee7ec3 --- /dev/null +++ b/doc/release-notes/7618-file-level-permissions-restricted-draft.md @@ -0,0 +1,3 @@ +File level permissions: Restricted files in Draft will now show a "Draft/Unpublished" tag in the UI when granting file access + +See #7618 diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java index 45604a5472b..bf790d3cc67 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java @@ -303,7 +303,18 @@ public Boolean getDeleted() { public void setDeleted(Boolean deleted) { this.deleted = deleted; } - + + @Transient + private boolean unpublished; + + public boolean getUnpublished() { + return unpublished; + } + + public void setUnpublished(boolean unpublished) { + this.unpublished = unpublished; + } + /* For use during file upload so that the user may delete files that have already been uploaded to the current dataset version @@ -596,7 +607,10 @@ public FileMetadata getLatestFileMetadata() { FileMetadata resultFileMetadata = null; if (fileMetadatas.size() == 1) { + setUnpublished(fileMetadatas.get(0).getDatasetVersion() == null || fileMetadatas.get(0).getDatasetVersion().getVersionState().equals(VersionState.DRAFT)); return fileMetadatas.get(0); + } else { + setUnpublished(false); // Since only one can be in Draft assume there is a published version } for (FileMetadata fileMetadata : fileMetadatas) { diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index d527ba3eeeb..ed16dea49c2 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1241,6 +1241,7 @@ dataverse.permissionsFiles.files.invalidMsg=There are no restricted files in thi dataverse.permissionsFiles.files.requested=Requested Files dataverse.permissionsFiles.files.selected=Selecting {0} of {1} {2} dataverse.permissionsFiles.files.includeDeleted=Include Deleted Files +dataverse.permissionsFiles.files.draftUnpublished=Draft/Unpublished dataverse.permissionsFiles.viewRemoveDialog.header=File Access dataverse.permissionsFiles.viewRemoveDialog.removeBtn=Remove Access dataverse.permissionsFiles.viewRemoveDialog.removeBtn.confirmation=Are you sure you want to remove access to this file? Once access has been removed, the user or group will no longer be able to download this file. diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml index 4e4e56f2051..2c8ff601acd 100644 --- a/src/main/webapp/permissions-manage-files.xhtml +++ b/src/main/webapp/permissions-manage-files.xhtml @@ -335,6 +335,7 @@ + From 1f7324d50f045e2b6895f8ada0499fe35e271870 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 8 Sep 2025 11:36:57 -0400 Subject: [PATCH 171/634] deploy rc to demo but after the 5 day wait #11790 --- .../source/developers/making-releases.rst | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index 4c9fd28b678..fb0cc630c6b 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -143,13 +143,6 @@ Deploy Release Candidate to QA Deploy the same war file to https://qa.dataverse.org using the same upgrade instructions as above. -Deploy Release Candidate to Demo --------------------------------- - -|dedicated| - -Deploy the same war file to https://demo.dataverse.org using the same upgrade instructions as above. - Solicit Feedback from Curation Team ----------------------------------- @@ -170,6 +163,15 @@ Make sure the guides directory appears in the expected location such as https:// When previewing the HTML version of docs from pull requests, we don't usually use this Jenkins job, relying instead on automated ReadTheDocs builds. The reason for doing this step now while we wait for feedback from the Curation Team is that it's an excellent time to fix the Jenkins job, if necessary, to accomodate any changes needed to continue to build the docs. For example, Sphinx might need to be updated or a dependency might need to be installed. Such changes should be listed in the release notes for documentation writers. +Deploy Release Candidate to Demo +-------------------------------- + +|dedicated| + +Time has passed. The curation team has given feedback. We've finished regression and performance testing. Fixes may have been merged into the "develop" branch. We're ready to actually make the release now, which includes deploying a release candidate to the demo server. + +Build a new war file, if necessary, and deploy it to https://demo.dataverse.org using the upgrade instructions in the release notes. + Merge Release Notes (Once Ready) -------------------------------- From 4c0e6eda810f661395cdc09f1f473fe6a42eff06 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 8 Sep 2025 11:38:42 -0400 Subject: [PATCH 172/634] typos #11790 --- doc/sphinx-guides/source/developers/making-releases.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index fb0cc630c6b..e7287e12b62 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -72,7 +72,7 @@ The benefits of the code freeze are: In short, the steps described below become easier under a code freeze. -Note: for a hotfix, a code freeze is necessary not because we want code to stop changing in the branch being hotfix released, but because bumping the version used in Jenkins/Ansible means that API tests will fail in pull requests until the version is bumped in those pull requests. Basically, we want to get the hotfix merged quickly so we can propogate the version bump into all open pull requests so that API tests can start passing again in those pull requests. +Note: for a hotfix, a code freeze is necessary not because we want code to stop changing in the branch being hotfix released, but because bumping the version used in Jenkins/Ansible means that API tests will fail in pull requests until the version is bumped in those pull requests. Basically, we want to get the hotfix merged quickly so we can propagate the version bump into all open pull requests so that API tests can start passing again in those pull requests. Push Back Milestones on Pull Requests That Missed the Train ----------------------------------------------------------- @@ -161,7 +161,7 @@ Click "Save" then "Build Now". Make sure the guides directory appears in the expected location such as https://guides.dataverse.org/en/6.8-rc.1/ -When previewing the HTML version of docs from pull requests, we don't usually use this Jenkins job, relying instead on automated ReadTheDocs builds. The reason for doing this step now while we wait for feedback from the Curation Team is that it's an excellent time to fix the Jenkins job, if necessary, to accomodate any changes needed to continue to build the docs. For example, Sphinx might need to be updated or a dependency might need to be installed. Such changes should be listed in the release notes for documentation writers. +When previewing the HTML version of docs from pull requests, we don't usually use this Jenkins job, relying instead on automated ReadTheDocs builds. The reason for doing this step now while we wait for feedback from the Curation Team is that it's an excellent time to fix the Jenkins job, if necessary, to accommodate any changes needed to continue to build the docs. For example, Sphinx might need to be updated or a dependency might need to be installed. Such changes should be listed in the release notes for documentation writers. Deploy Release Candidate to Demo -------------------------------- From fb3af17a4bdefdfa92dfe2588e062535eb74144f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 8 Sep 2025 11:41:38 -0400 Subject: [PATCH 173/634] move perf and reg testing to after rc deploy to qa #11790 --- .../source/developers/making-releases.rst | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index e7287e12b62..0da4d9e0a72 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -81,22 +81,6 @@ As of this writing, we optimistically add milestones to issues and pull requests Hopefully, as the release approached, the team has already decided which pull requests (that aren't related to the release) won't make the cut. If not, go ahead and bump them to the next release. -Conduct Performance Testing ---------------------------- - -|dedicated| - -See :doc:`/qa/performance-tests` for details. - -Conduct Regression Testing ---------------------------- - -|dedicated| - -Regression testing should be conducted on production data. -See :doc:`/qa/testing-approach` for details. -Refer to the provided regression checklist for the list of items to verify during the testing process: `Regression Checklist `_. - .. _write-release-notes: Write Release Notes @@ -148,6 +132,23 @@ Solicit Feedback from Curation Team Ask the curation team to test on https://qa.dataverse.org and give them five days to provide feedback. + +Conduct Performance Testing +--------------------------- + +|dedicated| + +See :doc:`/qa/performance-tests` for details. + +Conduct Regression Testing +--------------------------- + +|dedicated| + +Regression testing should be conducted on production data. +See :doc:`/qa/testing-approach` for details. +Refer to the provided regression checklist for the list of items to verify during the testing process: `Regression Checklist `_. + Build the Guides for the Release Candidate ------------------------------------------ From 238db74dca3de4dfd0b6b68c828430bdd5c000ae Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 8 Sep 2025 14:10:38 -0400 Subject: [PATCH 174/634] #11710 move logic in json printer --- .../harvard/iq/dataverse/api/Dataverses.java | 2 +- .../iq/dataverse/util/json/JsonPrinter.java | 90 +++++++++---------- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 97bc961c6b2..b522d63bcb0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -716,7 +716,7 @@ public Response getDataverse(@Context ContainerRequestContext crc, @PathParam("i return response(req -> { Dataverse dataverse = execCommand(new GetDataverseCommand(req, findDataverseOrDie(idtf))); boolean hideEmail = settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false); - return ok(json(dataverse, hideEmail, returnOwners, returnChildCount ? dataverseService.getChildCount(dataverse) : null)); + return ok(json(dataverse, hideEmail, returnOwners, false, returnChildCount ? dataverseService.getChildCount(dataverse) : null)); }, getRequestUser(crc)); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 779cb84e9cc..ae84ef18ec7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -293,65 +293,65 @@ public static JsonObjectBuilder json(Workflow wf){ public static JsonObjectBuilder json(Dataverse dv, boolean minimal) { if (!minimal){ - return json(dv, false, false, null); + return json(dv, false, false, false, null); } else { - return jsonObjectBuilder() - .add("id", dv.getId()) - .add("alias", dv.getAlias()) - .add("name", dv.getName()); + return json(dv, false, false, true, null); } } public static JsonObjectBuilder json(Dataverse dv) { - return json(dv, false, false, null); + return json(dv, false, false, false, null); } //TODO: Once we upgrade to Java EE 8 we can remove objects from the builder, and this email removal can be done in a better place. - public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean returnOwners, Long childCount) { + public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean returnOwners, Boolean minimal, Long childCount) { JsonObjectBuilder bld = jsonObjectBuilder() .add("id", dv.getId()) .add("alias", dv.getAlias()) - .add("name", dv.getName()) - .add("affiliation", dv.getAffiliation()); - if(!hideEmail) { - bld.add("dataverseContacts", JsonPrinter.json(dv.getDataverseContacts())); - } - if (returnOwners){ - bld.add("isPartOf", getOwnersFromDvObject(dv)); - } - bld.add("permissionRoot", dv.isPermissionRoot()) - .add("description", dv.getDescription()) - .add("dataverseType", dv.getDataverseType().name()) - .add("isMetadataBlockRoot", dv.isMetadataBlockRoot()) - .add("isFacetRoot", dv.isFacetRoot()); - if (dv.getOwner() != null) { - bld.add("ownerId", dv.getOwner().getId()); - } - if (dv.getCreateDate() != null) { - bld.add("creationDate", Util.getDateTimeFormat().format(dv.getCreateDate())); - } - if (dv.getDataverseTheme() != null) { - bld.add("theme", JsonPrinter.json(dv.getDataverseTheme())); - } - if(dv.getStorageDriverId() != null) { - bld.add("storageDriverLabel", DataAccess.getStorageDriverLabelFor(dv.getStorageDriverId())); - } - if (dv.getFilePIDsEnabled() != null) { - bld.add("filePIDsEnabled", dv.getFilePIDsEnabled()); - } - bld.add("effectiveRequiresFilesToPublishDataset", dv.getEffectiveRequiresFilesToPublishDataset()); - bld.add("isReleased", dv.isReleased()); + .add("name", dv.getName()); + //minimal refers to only returning the id alias and name for + //use in selecting collections available for linking + if (!minimal) { + bld.add("affiliation", dv.getAffiliation()); + if (!hideEmail) { + bld.add("dataverseContacts", JsonPrinter.json(dv.getDataverseContacts())); + } + if (returnOwners) { + bld.add("isPartOf", getOwnersFromDvObject(dv)); + } + bld.add("permissionRoot", dv.isPermissionRoot()) + .add("description", dv.getDescription()) + .add("dataverseType", dv.getDataverseType().name()) + .add("isMetadataBlockRoot", dv.isMetadataBlockRoot()) + .add("isFacetRoot", dv.isFacetRoot()); + if (dv.getOwner() != null) { + bld.add("ownerId", dv.getOwner().getId()); + } + if (dv.getCreateDate() != null) { + bld.add("creationDate", Util.getDateTimeFormat().format(dv.getCreateDate())); + } + if (dv.getDataverseTheme() != null) { + bld.add("theme", JsonPrinter.json(dv.getDataverseTheme())); + } + if (dv.getStorageDriverId() != null) { + bld.add("storageDriverLabel", DataAccess.getStorageDriverLabelFor(dv.getStorageDriverId())); + } + if (dv.getFilePIDsEnabled() != null) { + bld.add("filePIDsEnabled", dv.getFilePIDsEnabled()); + } + bld.add("effectiveRequiresFilesToPublishDataset", dv.getEffectiveRequiresFilesToPublishDataset()); + bld.add("isReleased", dv.isReleased()); - List inputLevels = dv.getDataverseFieldTypeInputLevels(); - if(!inputLevels.isEmpty()) { - bld.add("inputLevels", JsonPrinter.jsonDataverseFieldTypeInputLevels(inputLevels)); - } + List inputLevels = dv.getDataverseFieldTypeInputLevels(); + if (!inputLevels.isEmpty()) { + bld.add("inputLevels", JsonPrinter.jsonDataverseFieldTypeInputLevels(inputLevels)); + } - if (childCount != null) { - bld.add("childCount", childCount); + if (childCount != null) { + bld.add("childCount", childCount); + } + addDatasetFileCountLimit(dv, bld); } - addDatasetFileCountLimit(dv, bld); - return bld; } From 56e91ff33351944ae5c1faff6b486e71dd6887d6 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 8 Sep 2025 14:37:59 -0400 Subject: [PATCH 175/634] initial edits --- .../source/developers/big-data-support.rst | 143 ++++++++++++++++-- 1 file changed, 132 insertions(+), 11 deletions(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 75a50e2513d..ab8b92dff0f 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -1,15 +1,51 @@ -Big Data Support -================ +Scaling Dataverse with Data Size +================================ -Big data support includes some experimental options. Eventually more of this content will move to the Installation Guide. +This guide is intended to help administrators configure Dataverse appropriately to handle the amount of data their installation needs to manage. + +Scaling is a complex subject and there are many options available in Dataverse that can improve performance with larger scale data, but some of these +work differently than Dataverse's default configuration, potentially requiring user education, and can require additional expertise to manage. + +In general, there are three dimensions in which Dataverse can scale: + +1. **Number of Datasets** +2. **Number of Files per Dataset** +3. **Storage size of individual files and aggregate storage size** + +This guide will primarily focus on the latter two dimensions. .. contents:: |toctitle| :local: -Various components will need to be installed and/or configured for big data support via the methods described below. +Choosing the Right Approach +--------------------------- + +The table below provides guidance on which storage solution to choose based on your requirements: + ++------------------------+---------------------------+---------------------------+---------------------------+ +| **Solution** | **File Size Range** | **File Volume Support** | **Key Benefits** | ++========================+===========================+===========================+===========================+ +| Default Storage | Up to ~2GB | Up to thousands | Simple, no configuration | ++------------------------+---------------------------+---------------------------+---------------------------+ +| S3 Direct Upload | Up to several TB | Up to tens of thousands | Simple configuration, | +| | | | familiar user interface | ++------------------------+---------------------------+---------------------------+---------------------------+ +| Remote Storage | Unlimited | Unlimited | No file transfer needed, | +| | | | files remain in place | ++------------------------+---------------------------+---------------------------+---------------------------+ +| Globus Transfer | Unlimited | Millions+ | Robust transfer, | +| | | | resumable, high-speed | ++------------------------+---------------------------+---------------------------+---------------------------+ + +Handling Large Individual Files +------------------------------- + +When individual files exceed typical web upload/download capabilities (generally 1-2GB), you'll need specialized approaches. S3 Direct Upload and Download ------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Best for:** Files up to several TB in size with minimal configuration changes A lightweight option for supporting file sizes beyond a few gigabytes - a size that can cause performance issues when uploaded through a Dataverse installation itself - is to configure an S3 store to provide direct upload and download via 'pre-signed URLs'. When these options are configured, file uploads and downloads are made directly to and from a configured S3 store using secure (https) connections that enforce a Dataverse installation's access controls. (The upload and download URLs are signed with a unique key that only allows access for a short time period and a Dataverse installation will only generate such a URL if the user has permission to upload/download the specific file in question.) @@ -21,7 +57,6 @@ To configure these options, an administrator must set two JVM options for the Da ``./asadmin create-jvm-options "-Ddataverse.files..upload-redirect=true"`` - With multiple stores configured, it is possible to configure one S3 store with direct upload and/or download to support large files (in general or for specific Dataverse collections) while configuring only direct download, or no direct access for another store. The direct upload option now switches between uploading the file in one piece (up to 1 GB by default) and sending it as multiple parts. The default can be changed by setting: @@ -39,7 +74,7 @@ At present, one potential drawback for direct-upload is that files are only part .. _s3-direct-upload-features-disabled: Features that are Disabled if S3 Direct Upload is Enabled -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The following features are disabled when S3 direct upload is enabled. @@ -52,7 +87,7 @@ The following features are disabled when S3 direct upload is enabled. .. _cors-s3-bucket: Allow CORS for S3 Buckets -~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^ **IMPORTANT:** One additional step that is required to enable direct uploads via a Dataverse installation and for direct download to work with previewers and direct upload to work with dvwebloader (:ref:`folder-upload`) is to allow cross site (CORS) requests on your S3 store. The example below shows how to enable CORS rules (to support upload and download) on a bucket using the AWS CLI command line tool. Note that you may want to limit the AllowedOrigins and/or AllowedHeaders further. https://github.com/gdcc/dataverse-previewers/wiki/Using-Previewers-with-download-redirects-from-S3 has some additional information about doing this. @@ -85,12 +120,14 @@ Alternatively, you can enable CORS using the AWS S3 web interface, using json-en .. _s3-tags-and-direct-upload: S3 Tags and Direct Upload -~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^ Since the direct upload mechanism creates the final file rather than an intermediate temporary file, user actions, such as neither saving or canceling an upload session before closing the browser page, can leave an abandoned file in the store. The direct upload mechanism attempts to use S3 tags to aid in identifying/removing such files. Upon upload, files are given a "dv-state":"temp" tag which is removed when the dataset changes are saved and new files are added in the Dataverse installation. Note that not all S3 implementations support tags. Minio, for example, does not. With such stores, direct upload may not work and you might need to disable tagging. For details, see :ref:`s3-tagging` in the Installation Guide. Trusted Remote Storage with the ``remote`` Store Type ------------------------------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Best for:** Very large files that should remain in their original location For very large, and/or very sensitive data, it may not make sense to transfer or copy files to Dataverse at all. The experimental ``remote`` store type in the Dataverse software now supports this use case. @@ -151,7 +188,9 @@ To configure the options mentioned above, an administrator must set two JVM opti .. _globus-support: Globus File Transfer --------------------- +~~~~~~~~~~~~~~~~~~~ + +**Best for:** Very large files and high-performance computing environments Note: Globus file transfer is still experimental but feedback is welcome! See :ref:`support`. @@ -190,3 +229,85 @@ An overview of the control and data transfer interactions between components was See also :ref:`Globus settings <:GlobusSettings>`. An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. Due to its experimental nature it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag (see :ref:`feature-flags`) and the JVM option :ref:`dataverse.files.globus-monitoring-server`. + +Handling High Volume of Files +---------------------------- + +When dealing with datasets containing thousands or millions of files, different challenges arise beyond just file size. + +Performance Considerations +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Database Impact** + +Each file in Dataverse requires database entries for metadata, permissions, and other attributes. When dealing with very large numbers of files: + +* Database query performance may degrade +* Indexing operations take longer +* Dataset versioning becomes more resource-intensive + +**Recommended Approaches** + +For datasets with extremely high file counts (10,000+), consider these strategies: + +1. **Use hierarchical organization** - Group files into logical directories to improve navigation +2. **Batch operations** - Use the API for batch uploads rather than the UI +3. **Consider file bundling** - Where appropriate, bundle related small files into archives +4. **Use Globus for bulk transfers** - Globus is optimized for handling large numbers of files + +API-Based Batch Operations +~~~~~~~~~~~~~~~~~~~~~~~~~ + +For programmatically handling large numbers of files, Dataverse provides API endpoints that support batch operations: + +* The `/api/datasets/:id/addFiles` endpoint allows adding multiple files in a single request +* The `/api/datasets/:id/deleteFiles` endpoint allows removing multiple files at once + +See the :doc:`/api/native-api` documentation for details on these endpoints. + +Monitoring and Maintenance +~~~~~~~~~~~~~~~~~~~~~~~~~ + +When working with high file volumes: + +* Monitor database performance regularly +* Consider implementing scheduled maintenance windows for index optimization +* Use the Dataverse metrics API to track system performance + +Storage Strategy Recommendations +------------------------------- + +Based on both file size and volume considerations, here are some general recommendations: + +1. **For research projects with moderate data (< 2GB files, < 1000 files):** + * Default Dataverse storage is sufficient + +2. **For projects with large files but moderate file counts:** + * Configure S3 direct upload/download + * Set appropriate ingest size limits + +3. **For projects with very large files or sensitive data that should remain in place:** + * Use the remote storage option + * Consider implementing access controls at the remote storage level + +4. **For high-performance computing environments or very large datasets:** + * Implement Globus transfer + * Use batch operations via API + * Consider custom workflows for dataset creation and management + +5. **For datasets with millions of small files:** + * Consider file bundling where appropriate + * Use Globus for transfer + * Implement hierarchical organization + +Performance Tuning +----------------- + +Regardless of which storage solution you choose, consider these performance tuning options: + +* Increase JVM heap size for Dataverse application +* Optimize database indexes for file metadata tables +* Configure appropriate timeouts for large file transfers +* Consider dedicated storage nodes for high-volume installations + +For detailed performance tuning recommendations, see the :doc:`/installation/config` guide. From abc1d079d6c3040936f6bf8639eb17086f23be32 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 9 Sep 2025 13:40:03 +0100 Subject: [PATCH 176/634] Changed: required permission of ListDataverseTemplatesCommand to AddDataset --- .../engine/command/impl/ListDataverseTemplatesCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseTemplatesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseTemplatesCommand.java index 3e7d4664793..1dbb05e47b5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseTemplatesCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseTemplatesCommand.java @@ -14,7 +14,7 @@ /** * Lists the templates {@link Template} of a {@link Dataverse}. */ -@RequiredPermissions(Permission.EditDataverse) +@RequiredPermissions(Permission.AddDataset) public class ListDataverseTemplatesCommand extends AbstractCommand> { private final Dataverse dataverse; From 08761f4524d8513be47cd41b8e282496b6aa9afc Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 9 Sep 2025 14:13:05 +0100 Subject: [PATCH 177/634] Added: IT test cases for testCreateAndGetTemplates --- .../harvard/iq/dataverse/api/DataversesIT.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 4f9fbae7ffc..e0eaadb1e82 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.api; +import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.util.json.JsonParseException; import edu.harvard.iq.dataverse.util.json.JsonParser; import edu.harvard.iq.dataverse.util.json.JsonUtil; @@ -2288,9 +2289,11 @@ public void testUpdateInputLevelDisplayOnCreateOverride() { public void testCreateAndGetTemplates() { Response createUserResponse = UtilIT.createRandomUser(); String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); + String username = UtilIT.getUsernameFromResponse(createUserResponse); Response createSecondUserResponse = UtilIT.createRandomUser(); String secondApiToken = UtilIT.getApiTokenFromResponse(createSecondUserResponse); + String secondUsername = UtilIT.getUsernameFromResponse(createSecondUserResponse); Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); @@ -2368,9 +2371,19 @@ public void testCreateAndGetTemplates() { .body("data[0].instructions[0].instructionText", equalTo("The author data")) .body("data[0].dataverseAlias", equalTo(dataverseAlias)); - // Templates retrieval should fail if the user lacks dataverse edit permissions + // Templates retrieval should fail if a secondary user lacks dataset creation permissions + getTemplateResponse = UtilIT.getTemplates(dataverseAlias, secondApiToken); getTemplateResponse.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode()); + + // Template retrieval should succeed if the secondary user has dataset creation permissions + + UtilIT.setSuperuserStatus(username, true); + Response grantRoleResponse = UtilIT.grantRoleOnDataverse(dataverseAlias, DataverseRole.DS_CONTRIBUTOR, "@" + secondUsername, apiToken); + grantRoleResponse.then().assertThat().statusCode(OK.getStatusCode()); + + getTemplateResponse = UtilIT.getTemplates(dataverseAlias, secondApiToken); + getTemplateResponse.then().assertThat().statusCode(OK.getStatusCode()); } private String getSuperuserToken() { From 10de15a1008be3e22d528db2b746b91a1a8c6e61 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 9 Sep 2025 14:14:42 +0100 Subject: [PATCH 178/634] Fixed: typo in test comment --- src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index e0eaadb1e82..ffe32c178ab 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -2376,7 +2376,7 @@ public void testCreateAndGetTemplates() { getTemplateResponse = UtilIT.getTemplates(dataverseAlias, secondApiToken); getTemplateResponse.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode()); - // Template retrieval should succeed if the secondary user has dataset creation permissions + // Templates retrieval should succeed if the secondary user has dataset creation permissions UtilIT.setSuperuserStatus(username, true); Response grantRoleResponse = UtilIT.grantRoleOnDataverse(dataverseAlias, DataverseRole.DS_CONTRIBUTOR, "@" + secondUsername, apiToken); From d03ddcdc2817405414fdf1ed5153b0070070810a Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 9 Sep 2025 14:18:01 +0100 Subject: [PATCH 179/634] Added: release notes for #11796 --- ...796-fix-list-dataverse-templates-api-permissions.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/release-notes/11796-fix-list-dataverse-templates-api-permissions.md diff --git a/doc/release-notes/11796-fix-list-dataverse-templates-api-permissions.md b/doc/release-notes/11796-fix-list-dataverse-templates-api-permissions.md new file mode 100644 index 00000000000..e15f9f4dfa1 --- /dev/null +++ b/doc/release-notes/11796-fix-list-dataverse-templates-api-permissions.md @@ -0,0 +1,10 @@ +### Fixed: Permissions for `/{identifier}/templates` endpoint + +The `/{identifier}/templates` endpoint previously required **`editDataverse`** permissions to retrieve the list of dataverse templates. + +This has been corrected: the endpoint now requires **`addDataset`** permissions instead. + +**Impact:** +- The endpoint now works in scenarios such as the **Create Dataset form** in the SPA UI, without needing unnecessary elevated permissions. + +Related issues: #11796 \ No newline at end of file From 5791664ad4d69b1573ac09d3ce3492a0bce8c816 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 10 Sep 2025 09:49:15 -0400 Subject: [PATCH 180/634] #11710 fix comment from copy --- src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 8b0f5323659..d0ae3f4d8b9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -2455,7 +2455,7 @@ public void testCreateAndGetTemplates() throws JsonParseException { String[] newFacetIds = new String[]{"contributorName"}; String[] newMetadataBlockNames = new String[]{"citation", "geospatial", "biomedical"}; - // Assert that the error is returned for having both MetadataBlockNames and inheritMetadataBlocksFromParent + //Giving the new Dataverse updated metadatablocks so that it will not inherit templates Response updateDataverseResponse = UtilIT.updateDataverse( dataverseAlias, dataverseAlias, newName, newAffiliation, newDataverseType, newContactEmails, newInputLevelNames, null, newMetadataBlockNames, apiToken, From ac8bdc17020bd87193b3d4fe0431a6804fbf2e6d Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 10 Sep 2025 10:09:44 -0400 Subject: [PATCH 181/634] fix for #11800 hide guestbook when not used, add missing method to FilePage/file.xhtml --- src/main/java/edu/harvard/iq/dataverse/FilePage.java | 5 +++++ src/main/webapp/file.xhtml | 2 ++ src/main/webapp/guestbook-terms-popup-fragment.xhtml | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index 50ebbc33634..dd1bd56d5bd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -420,6 +420,11 @@ public boolean isGuestbookPopupRequiredAtDownload(){ DatasetVersion workingVersion = fileMetadata.getDatasetVersion(); return FileUtil.isGuestbookPopupRequired(workingVersion) && !workingVersion.getDataset().getEffectiveGuestbookEntryAtRequest(); } + + public boolean isGuestbookPopupRequired(){ + DatasetVersion workingVersion = fileMetadata.getDatasetVersion(); + return FileUtil.isGuestbookPopupRequired(workingVersion); + } public void setFileMetadata(FileMetadata fileMetadata) { this.fileMetadata = fileMetadata; diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index 8a5c2709e5d..2292ebf4c45 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -374,6 +374,7 @@ + @@ -681,6 +682,7 @@ + diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml index 7d157fcb72c..a17145a43ce 100644 --- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml +++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml @@ -142,7 +142,7 @@ value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.termsOfAccess)}" escape="false" /> - +
      From 811f178c3870dff853b257348058404061cdacbc Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 10 Sep 2025 10:52:45 -0400 Subject: [PATCH 182/634] release note --- doc/release-notes/11800-qb-ra-fixes.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 doc/release-notes/11800-qb-ra-fixes.md diff --git a/doc/release-notes/11800-qb-ra-fixes.md b/doc/release-notes/11800-qb-ra-fixes.md new file mode 100644 index 00000000000..5c7ef82f7f2 --- /dev/null +++ b/doc/release-notes/11800-qb-ra-fixes.md @@ -0,0 +1,2 @@ +This release fixes problems with guestbook questions being displayed at download when files are selected from the dataset files table +when guestbook-at-request is enabled and not displaying when they should when access is requested from the file page. \ No newline at end of file From cfb841cf049b71934f7276e8d96067325f14d745 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 10 Sep 2025 10:57:28 -0400 Subject: [PATCH 183/634] #11710 handle guest users --- .../edu/harvard/iq/dataverse/api/Dataverses.java | 13 ++++++++++--- .../edu/harvard/iq/dataverse/api/DataversesIT.java | 5 +++++ .../java/edu/harvard/iq/dataverse/api/UtilIT.java | 12 +++++++++--- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index b522d63bcb0..16415e0c155 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1786,10 +1786,18 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P DvObject dvObject = findDvoByIdAndTypeOrDie(dvIdtf, type, false); List dataversesForLinkingSearch = new ArrayList(); - AuthenticatedUser requestUser = (AuthenticatedUser) getRequestUser(crc); + User requestUser = (User) getRequestUser(crc); + AuthenticatedUser authUser; + if (!requestUser.isAuthenticated()) { + throw new WrappedResponse(error(Response.Status.BAD_REQUEST, + BundleUtil.getStringFromBundle("dataverse.link.user"))); + } else { + authUser = (AuthenticatedUser) requestUser; + } + DataverseRequest dvReq = new DataverseRequest(requestUser, (IpAddress) null); List dataversesForLinking; - dataversesForLinking = permissionService.findPermittedCollections(dvReq, requestUser, Permission.LinkDataset); + dataversesForLinking = permissionService.findPermittedCollections(dvReq, authUser, Permission.LinkDataset); List mergedWithSearch = new ArrayList<>(); dataversesForLinking = dataverseService.removeUnlinkableDataverses(dataversesForLinking, dvObject); @@ -1820,7 +1828,6 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P return wr.getResponse(); } catch (Exception e) { return error(Status.BAD_REQUEST, e.getLocalizedMessage()); - } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index d0ae3f4d8b9..a47672076b3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -766,6 +766,11 @@ public void testGetLinkableDataverses(){ .statusCode(OK.getStatusCode()) .body("data.size()", equalTo(0)); + Response getGuestUnavailableForDataset = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, "", dataverseAliasUnavailableForLinking); + getGuestUnavailableForDataset.prettyPrint(); + getGuestUnavailableForDataset.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()); + Response getUnavailableForDataverse = UtilIT.getLinkableDataverses("dataverse", dataverseAlias, apiToken, dataverseAliasUnavailableForLinking); getUnavailableForDataverse.prettyPrint(); getUnavailableForDataverse.then().assertThat() diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 0eb4a79a3eb..00dbbd7dada 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4729,10 +4729,16 @@ static Response getLinkableDataverses (String type, String dvObjectId, String ap optionalQueryParam = "&persistentId=" + dvObjectId; } } + + if (!apiToken.isEmpty()) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/dataverses/" + idInPath + "/" + type + "/linkingDataverses?searchTerm=" + searchTerm + optionalQueryParam); - return given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .get("/api/dataverses/" + idInPath + "/" + type + "/linkingDataverses?searchTerm=" + searchTerm + optionalQueryParam); + } else { + return given() + .get("/api/dataverses/" + idInPath + "/" + type + "/linkingDataverses?searchTerm=" + searchTerm + optionalQueryParam); + } } static Response updateDataverseFeaturedItem(long featuredItemId, From 26661fc17bb2e53bc1aff4dc1e51f1443e2e820c Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 10 Sep 2025 11:50:26 -0400 Subject: [PATCH 184/634] #11710 fix pass bad alias --- .../edu/harvard/iq/dataverse/api/AbstractApiBean.java | 3 +++ .../edu/harvard/iq/dataverse/api/DataversesIT.java | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 938d7fc9081..12cbe045f8c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -637,6 +637,9 @@ protected DvObject findDvoByIdAndTypeOrDie(@NotNull final String dvIdtf, String if (testForReleased){ DataverseFeaturedItem.validateTypeAndDvObject(dvIdtf, dvObject, dvType); } + if (dvObject == null) { + throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.dvo.error.dvObjectNotFound", Collections.singletonList(dvIdtf)))); + } return dvObject; } catch (IllegalArgumentException e) { throw new WrappedResponse(error(Response.Status.BAD_REQUEST, e.getMessage())); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index a47672076b3..9657ebd8fd9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -740,8 +740,15 @@ public void testGetLinkableDataverses(){ getLinkableDataversesForDataversePartial.prettyPrint(); getLinkableDataversesForDataversePartial.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data[0].alias", equalTo(dataverseAliasForLinking)); - + .body("data[0].alias", equalTo(dataverseAliasForLinking)); + + //Try with bad target alias + searchTerm = ""; + Response getLinkableDataversesForDataverseBadId = UtilIT.getLinkableDataverses("dataverse", "junque@#", apiToken, searchTerm); + getLinkableDataversesForDataverseBadId.prettyPrint(); + getLinkableDataversesForDataverseBadId.then().assertThat() + .statusCode(NOT_FOUND.getStatusCode()); + // create new user and dataverse - the new dataverse should not be available to the first user for linking... Response createUserTwo = UtilIT.createRandomUser(); String apiTokenTwo = UtilIT.getApiTokenFromResponse(createUserTwo); From 886256c0da2b7f41a3773a572a9684b53fd2c2d9 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 10 Sep 2025 13:48:16 -0400 Subject: [PATCH 185/634] #11710 remove parents from available linking DV --- .../iq/dataverse/DataverseServiceBean.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index bc91cb4e081..57bc0bc5450 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -572,13 +572,21 @@ public List removeUnlinkableDataverses(List allWithPerms, remove.add(removeIt); }); } + if (dvo instanceof Dataverse dataverse) { remove.add(dataverse); - } else { - //dataset is always owned by a dataverse - remove.add((Dataverse) dvo.getOwner()); - } + } + + DvObject testDVO = dvo; + //Remove DVO's parent up to Root + while (testDVO != null) { + if (testDVO.getOwner() == null) { + break; // we are at the root; which by definition is metadata block root, regardless of the value + } + remove.add((Dataverse) testDVO.getOwner()); + testDVO = testDVO.getOwner(); + } for (Dataverse res : allWithPerms) { if (!remove.contains(res)) { From 76333aaf50b1d6964aea321300c8b332ee4037a4 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 10 Sep 2025 14:34:21 -0400 Subject: [PATCH 186/634] #11710 allow search term to be null --- .../harvard/iq/dataverse/api/Dataverses.java | 2 +- .../iq/dataverse/api/DataversesIT.java | 9 ++++- .../edu/harvard/iq/dataverse/api/UtilIT.java | 38 ++++++++++++++----- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 16415e0c155..9e78f527258 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1803,7 +1803,7 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P dataversesForLinking = dataverseService.removeUnlinkableDataverses(dataversesForLinking, dvObject); //Only do search lookup if search term is there. Otherwise just include the collections based on perms - if (!searchTerm.isEmpty()) { + if (searchTerm != null && !searchTerm.isEmpty()) { dataversesForLinkingSearch = dataverseService.filterDataversesByNamePattern(searchTerm); if (!dataversesForLinkingSearch.isEmpty()) { for (Dataverse dv : dataversesForLinking) { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 9657ebd8fd9..2333faccf79 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -740,7 +740,14 @@ public void testGetLinkableDataverses(){ getLinkableDataversesForDataversePartial.prettyPrint(); getLinkableDataversesForDataversePartial.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data[0].alias", equalTo(dataverseAliasForLinking)); + .body("data[0].alias", equalTo(dataverseAliasForLinking)); + + //Should get same result if search term is null + getLinkableDataversesForDataversePartial = UtilIT.getLinkableDataverses("dataverse", dataverseAlias, apiToken, null); + getLinkableDataversesForDataversePartial.prettyPrint(); + getLinkableDataversesForDataversePartial.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].alias", equalTo(dataverseAliasForLinking)); //Try with bad target alias searchTerm = ""; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 79a71441e26..5f5416e08cc 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4729,28 +4729,46 @@ static Response updateDataverseFeaturedItem(long featuredItemId, return updateDataverseFeaturedItem(featuredItemId, content, displayOrder, keepFile, pathToFile, null, null, apiToken); } - static Response getLinkableDataverses (String type, String dvObjectId, String apiToken, String searchTerm) { + static Response getLinkableDataverses(String type, String dvObjectId, String apiToken, String searchTerm) { String idInPath = dvObjectId; // Assume it's a number to start. String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. if (type.equals("dataset")) { if (!NumberUtils.isCreatable(idInPath)) { idInPath = ":persistentId"; - optionalQueryParam = "&persistentId=" + dvObjectId; + if (searchTerm == null) { + optionalQueryParam = "?persistentId=" + dvObjectId; + } else { + optionalQueryParam = "&persistentId=" + dvObjectId; + } } } - - if (!apiToken.isEmpty()) { - return given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .get("/api/dataverses/" + idInPath + "/" + type + "/linkingDataverses?searchTerm=" + searchTerm + optionalQueryParam); + + if (searchTerm == null) { + if (!apiToken.isEmpty()) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/dataverses/" + idInPath + "/" + type + "/linkingDataverses" + optionalQueryParam); + + } else { + return given() + .get("/api/dataverses/" + idInPath + "/" + type + "/linkingDataverses" + optionalQueryParam); + } } else { - return given() - .get("/api/dataverses/" + idInPath + "/" + type + "/linkingDataverses?searchTerm=" + searchTerm + optionalQueryParam); - } + if (!apiToken.isEmpty()) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/dataverses/" + idInPath + "/" + type + "/linkingDataverses?searchTerm=" + searchTerm + optionalQueryParam); + + } else { + return given() + .get("/api/dataverses/" + idInPath + "/" + type + "/linkingDataverses?searchTerm=" + searchTerm + optionalQueryParam); + } + } } + static Response updateDataverseFeaturedItem(long featuredItemId, String content, int displayOrder, From bafa0d7204ef46938b211de6309de8703b0358a4 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 10 Sep 2025 15:30:05 -0400 Subject: [PATCH 187/634] #11710 add test for linking child --- .../harvard/iq/dataverse/api/DataversesIT.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 2333faccf79..3fbd7cff23f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -690,6 +690,11 @@ public void testGetLinkableDataverses(){ //Create dataverse for linking Response createDataverseResponseForLinking = UtilIT.createRandomDataverse(apiToken); String dataverseAliasForLinking = UtilIT.getAliasFromResponse(createDataverseResponseForLinking); + + // Create a child of the linking DV + Response createLevel1a = UtilIT.createSubDataverse(UtilIT.getRandomDvAlias() + "-level1a", null, apiToken, dataverseAliasForLinking); + createLevel1a.prettyPrint(); + String childLinking = UtilIT.getAliasFromResponse(createLevel1a); Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); assertEquals(200, publishDataverse.getStatusCode()); @@ -747,7 +752,14 @@ public void testGetLinkableDataverses(){ getLinkableDataversesForDataversePartial.prettyPrint(); getLinkableDataversesForDataversePartial.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data[0].alias", equalTo(dataverseAliasForLinking)); + .body("data[0].alias", equalTo(dataverseAliasForLinking)); + + //Try to link the child - should not link to parent - should link to "uncle" + getLinkableDataversesForDataversePartial = UtilIT.getLinkableDataverses("dataverse", childLinking, apiToken, searchTerm); + getLinkableDataversesForDataversePartial.prettyPrint(); + getLinkableDataversesForDataversePartial.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].alias", equalTo(dataverseAlias)); //Try with bad target alias searchTerm = ""; @@ -827,6 +839,9 @@ public void testGetLinkableDataverses(){ Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); assertEquals(200, deleteDataverseResponse.getStatusCode()); + deleteDataverseResponse = UtilIT.deleteDataverse(childLinking, apiToken); + assertEquals(200, deleteDataverseResponse.getStatusCode()); + deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAliasForLinking, apiToken); assertEquals(200, deleteDataverseResponse.getStatusCode()); From 5774d950d2de93e778fd5799faff98ea127795d6 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 10 Sep 2025 15:50:55 -0400 Subject: [PATCH 188/634] #11710 remove generic exception --- src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 9e78f527258..a8743190114 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1826,9 +1826,8 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P return ok(dvBuilder); } catch (WrappedResponse wr) { return wr.getResponse(); - } catch (Exception e) { - return error(Status.BAD_REQUEST, e.getLocalizedMessage()); - } + } + } From ac935e853bd4bf22f1205e783d1bd19a0e25b804 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 10 Sep 2025 16:01:04 -0400 Subject: [PATCH 189/634] update docs to explain how to delete input levels --- doc/sphinx-guides/source/api/changelog.rst | 2 +- doc/sphinx-guides/source/api/native-api.rst | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 96629823673..b793981e761 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -13,7 +13,7 @@ v6.8 - For POST /api/files/{id}/metadata passing an empty string ("description":"") or array ("categories":[]) will no longer be ignored. Empty fields will now clear out the values in the file's metadata. To ignore the fields simply do not include them in the JSON string. - For PUT /api/datasets/{id}/editMetadata the query parameter "sourceInternalVersionNumber" has been removed and replaced with "sourceLastUpdateTime" to verify that the data being edited hasn't been modified and isn't stale. - For GET /api/dataverses/$dataverse-alias/links the Json response has changed breaking the backward compatibility of the API. -- For PUT /api/dataverses/$dataverse-alias/inputLevels custom input levels that had been previously set will no longer be deleted. +- For PUT /api/dataverses/$dataverse-alias/inputLevels custom input levels that had been previously set will no longer be deleted. To delete input levels send an empty list (deletes all), then send the new/modified list. - For GET /api/externalTools and /api/externalTools/{id} the responses are now formatted as JSON (previously the toolParameters and allowedApiCalls were a JSON object and array (respectively) that were serialized as JSON strings) and any configured "requirements" are included. v6.7 diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 35cd807ef96..65956904223 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1106,7 +1106,8 @@ Update Collection Input Levels Updates the dataset field type input levels in a collection. -Please note that this endpoint does not change previously updated input levels of the collection page, so if you want to keep the modify exisitng ones, you will need to include them in the JSON request body. +Please note that this endpoint does not change previously updated input levels of the collection page, so if you want to add new levels or modify existing ones, you will need to include them in the JSON request body. +In order to delete input levels you must call this API with an empty list to delete all of the input levels, then call this API with the new list of input levels. If one of the input levels corresponds to a dataset field type belonging to a metadata block that does not exist in the collection, the metadata block will be added to the collection. From 97fae8b8988b7febca885c3c775224aab76719bc Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 11 Sep 2025 08:32:16 -0400 Subject: [PATCH 190/634] #11710 combine search and perms query --- .../iq/dataverse/PermissionServiceBean.java | 56 +++++++++++++++++-- .../harvard/iq/dataverse/api/Dataverses.java | 13 ++++- 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index 70e8145835b..f6ef9a1d5bf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -105,7 +105,7 @@ WITH grouplist AS ( WHERE explicitgroup_authenticateduser.containedauthenticatedusers_id = @USERID ) - SELECT * FROM DATAVERSE WHERE id IN ( + SELECT * FROM DATAVERSE dv WHERE id IN ( SELECT definitionpoint_id FROM roleassignment WHERE roleassignment.assigneeidentifier IN ( @@ -161,7 +161,7 @@ AND EXISTS (SELECT id FROM dataverserole WHERE dataverserole.id = roleassignment AND @IPRANGESQL ) ) - ) + ) @SEARCHPREDICATE """; /** * A request-level permission query (e.g includes IP ras). @@ -921,12 +921,21 @@ private boolean hasUnrestrictedReleasedFiles(DatasetVersion targetDatasetVersion Long result = em.createQuery(criteriaQuery).getSingleResult(); return result > 0; } - + public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, Permission permission) { - return findPermittedCollections(request, user, 1 << permission.ordinal()); + return findPermittedCollections(request, user, 1 << permission.ordinal(), ""); + } + + public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, Permission permission, String searchTerm) { + return findPermittedCollections(request, user, 1 << permission.ordinal(), searchTerm); } public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit) { + return findPermittedCollections(request, user, permissionBit, ""); + } + + + public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit, String searchTerm) { if (user != null) { // IP Group - Only check IP if a User is calling for themself String ipRangeSQL = "FALSE"; @@ -959,15 +968,52 @@ public List findPermittedCollections(DataverseRequest request, Authen if (user.isSuperuser()) { sqlCode = LIST_ALL_DATAVERSES_SUPERUSER_HAS_PERMISSION; } else { + System.out.print("searchTerm: " + searchTerm); sqlCode = LIST_ALL_DATAVERSES_USER_HAS_PERMISSION .replace("@USERID", String.valueOf(user.getId())) .replace("@PERMISSIONBIT", String.valueOf(permissionBit)) - .replace("@IPRANGESQL", ipRangeSQL); + .replace("@IPRANGESQL", ipRangeSQL) + .replace("@SEARCHPREDICATE", getPredicateFromSearchTerm(searchTerm)); } + System.out.print(sqlCode); return em.createNativeQuery(sqlCode, Dataverse.class).getResultList(); } return null; } + + private String getPredicateFromSearchTerm(String searchTerm){ + + if (searchTerm == null || searchTerm.isEmpty()){ + System.out.print("search term null or empty"); + return ""; + } + + String pattern = searchTerm.toLowerCase(); + + String pattern1 = pattern + "%"; + String pattern2 = "% " + pattern + "%"; + + // Adjust the queries for very short, 1 and 2-character patterns: + if (pattern.length() == 1) { + pattern1 = pattern; + pattern2 = pattern + " %"; + } + /*if (pattern.length() == 2) { + pattern2 = pattern + "%"; + }*/ + + + String qstr = "and ((LOWER(dv.name) LIKE @DATAVERSE and ((SUBSTRING(LOWER(dv.name),0,(LENGTH(dv.name)-9)) LIKE @PATTERN1) " + + " or (SUBSTRING(LOWER(dv.name),0,(LENGTH(dv.name)-9)) LIKE @PATTERN2))) " + + " or (LOWER(dv.name) NOT LIKE @DATAVERSE and ((LOWER(dv.name) LIKE @PATTERN1) " + + " or (LOWER(dv.name) LIKE @PATTERN2)))) "; + qstr = qstr.replace("@DATAVERSE", "'%dataverse'") + .replace("@PATTERN1", "'" + pattern1 + "'") + .replace("@PATTERN2", "'" + pattern2 + "'"); + + System.out.print("Query String: " + qstr); + return qstr; + } /** * Calculates the complete list of role assignments for a given user on a DvObject. diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index a8743190114..3078afd7419 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1797,12 +1797,19 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P DataverseRequest dvReq = new DataverseRequest(requestUser, (IpAddress) null); List dataversesForLinking; - dataversesForLinking = permissionService.findPermittedCollections(dvReq, authUser, Permission.LinkDataset); + String searchParam; + if(searchTerm != null){ + searchParam = searchTerm; + } else { + searchParam = ""; + } + dataversesForLinking = permissionService.findPermittedCollections(dvReq, authUser, Permission.LinkDataset, searchParam); List mergedWithSearch = new ArrayList<>(); dataversesForLinking = dataverseService.removeUnlinkableDataverses(dataversesForLinking, dvObject); //Only do search lookup if search term is there. Otherwise just include the collections based on perms + /* if (searchTerm != null && !searchTerm.isEmpty()) { dataversesForLinkingSearch = dataverseService.filterDataversesByNamePattern(searchTerm); if (!dataversesForLinkingSearch.isEmpty()) { @@ -1816,7 +1823,9 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P //search term empty then add all based on perms mergedWithSearch.addAll(dataversesForLinking); } - + */ + mergedWithSearch.addAll(dataversesForLinking); + JsonArrayBuilder dvBuilder = Json.createArrayBuilder(); if (!mergedWithSearch.isEmpty()) { for (Dataverse dv : mergedWithSearch) { From ce946d02dcd8e73feb3ff8f4f487b1441ee3b8b1 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 11 Sep 2025 10:41:36 -0400 Subject: [PATCH 191/634] #11710 move dv creation for test --- .../edu/harvard/iq/dataverse/api/DataversesIT.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 3fbd7cff23f..d11d9158f2a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -691,10 +691,6 @@ public void testGetLinkableDataverses(){ Response createDataverseResponseForLinking = UtilIT.createRandomDataverse(apiToken); String dataverseAliasForLinking = UtilIT.getAliasFromResponse(createDataverseResponseForLinking); - // Create a child of the linking DV - Response createLevel1a = UtilIT.createSubDataverse(UtilIT.getRandomDvAlias() + "-level1a", null, apiToken, dataverseAliasForLinking); - createLevel1a.prettyPrint(); - String childLinking = UtilIT.getAliasFromResponse(createLevel1a); Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); assertEquals(200, publishDataverse.getStatusCode()); @@ -752,7 +748,12 @@ public void testGetLinkableDataverses(){ getLinkableDataversesForDataversePartial.prettyPrint(); getLinkableDataversesForDataversePartial.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data[0].alias", equalTo(dataverseAliasForLinking)); + .body("data[0].alias", equalTo(dataverseAliasForLinking)); + + // Create a child of the linking DV + Response createLevel1a = UtilIT.createSubDataverse(UtilIT.getRandomDvAlias() + "-level1a", null, apiToken, dataverseAliasForLinking); + createLevel1a.prettyPrint(); + String childLinking = UtilIT.getAliasFromResponse(createLevel1a); //Try to link the child - should not link to parent - should link to "uncle" getLinkableDataversesForDataversePartial = UtilIT.getLinkableDataverses("dataverse", childLinking, apiToken, searchTerm); From 3abc4d04d99cae36ff10d723e6018cf7bfc37149 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 11 Sep 2025 12:23:20 -0400 Subject: [PATCH 192/634] #11710 parametize the query --- .../iq/dataverse/PermissionServiceBean.java | 83 +++++++++---------- .../harvard/iq/dataverse/api/Dataverses.java | 1 - .../iq/dataverse/api/DataversesIT.java | 6 ++ 3 files changed, 46 insertions(+), 44 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index f6ef9a1d5bf..1fb0d52e9fc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -161,8 +161,16 @@ AND EXISTS (SELECT id FROM dataverserole WHERE dataverserole.id = roleassignment AND @IPRANGESQL ) ) - ) @SEARCHPREDICATE + ) """; + + private static final String SEARCH_PARAMS = """ + and ((LOWER(dv.name) LIKE ? and ((SUBSTRING(LOWER(dv.name),0,(LENGTH(dv.name)-9)) LIKE ?) + or (SUBSTRING(LOWER(dv.name),0,(LENGTH(dv.name)-9)) LIKE ?))) + or (LOWER(dv.name) NOT LIKE ? and ((LOWER(dv.name) LIKE ?) + or (LOWER(dv.name) LIKE ?)))) + """; + /** * A request-level permission query (e.g includes IP ras). */ @@ -968,52 +976,41 @@ public List findPermittedCollections(DataverseRequest request, Authen if (user.isSuperuser()) { sqlCode = LIST_ALL_DATAVERSES_SUPERUSER_HAS_PERMISSION; } else { - System.out.print("searchTerm: " + searchTerm); - sqlCode = LIST_ALL_DATAVERSES_USER_HAS_PERMISSION - .replace("@USERID", String.valueOf(user.getId())) - .replace("@PERMISSIONBIT", String.valueOf(permissionBit)) - .replace("@IPRANGESQL", ipRangeSQL) - .replace("@SEARCHPREDICATE", getPredicateFromSearchTerm(searchTerm)); + if (searchTerm == null || searchTerm.isEmpty()) { + sqlCode = LIST_ALL_DATAVERSES_USER_HAS_PERMISSION + .replace("@USERID", String.valueOf(user.getId())) + .replace("@PERMISSIONBIT", String.valueOf(permissionBit)) + .replace("@IPRANGESQL", ipRangeSQL); + return em.createNativeQuery(sqlCode, Dataverse.class).getResultList(); + } else { + String pattern = searchTerm.toLowerCase(); + String pattern1 = pattern + "%"; + String pattern2 = "% " + pattern + "%"; + + // Adjust the queries for very short, 1 + if (pattern.length() == 1) { + pattern1 = pattern; + pattern2 = pattern + " %"; + } + + sqlCode = LIST_ALL_DATAVERSES_USER_HAS_PERMISSION.concat(SEARCH_PARAMS) + .replace("@USERID", String.valueOf(user.getId())) + .replace("@PERMISSIONBIT", String.valueOf(permissionBit)) + .replace("@IPRANGESQL", ipRangeSQL); + + Query query = em.createNativeQuery(sqlCode, Dataverse.class); + query.setParameter(1, "%dataverse"); + query.setParameter(2, pattern1); + query.setParameter(3, pattern2); + query.setParameter(4, "%dataverse"); + query.setParameter(5, pattern1); + query.setParameter(6, pattern2); + return query.getResultList(); + } } - System.out.print(sqlCode); - return em.createNativeQuery(sqlCode, Dataverse.class).getResultList(); } return null; } - - private String getPredicateFromSearchTerm(String searchTerm){ - - if (searchTerm == null || searchTerm.isEmpty()){ - System.out.print("search term null or empty"); - return ""; - } - - String pattern = searchTerm.toLowerCase(); - - String pattern1 = pattern + "%"; - String pattern2 = "% " + pattern + "%"; - - // Adjust the queries for very short, 1 and 2-character patterns: - if (pattern.length() == 1) { - pattern1 = pattern; - pattern2 = pattern + " %"; - } - /*if (pattern.length() == 2) { - pattern2 = pattern + "%"; - }*/ - - - String qstr = "and ((LOWER(dv.name) LIKE @DATAVERSE and ((SUBSTRING(LOWER(dv.name),0,(LENGTH(dv.name)-9)) LIKE @PATTERN1) " - + " or (SUBSTRING(LOWER(dv.name),0,(LENGTH(dv.name)-9)) LIKE @PATTERN2))) " - + " or (LOWER(dv.name) NOT LIKE @DATAVERSE and ((LOWER(dv.name) LIKE @PATTERN1) " - + " or (LOWER(dv.name) LIKE @PATTERN2)))) "; - qstr = qstr.replace("@DATAVERSE", "'%dataverse'") - .replace("@PATTERN1", "'" + pattern1 + "'") - .replace("@PATTERN2", "'" + pattern2 + "'"); - - System.out.print("Query String: " + qstr); - return qstr; - } /** * Calculates the complete list of role assignments for a given user on a DvObject. diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 3078afd7419..0abfed3d4ba 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -70,7 +70,6 @@ import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.StreamingOutput; -import java.sql.PreparedStatement; import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index d11d9158f2a..e849a32a513 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -787,6 +787,12 @@ public void testGetLinkableDataverses(){ String apiTokenThree = UtilIT.getApiTokenFromResponse(createUserThree); String usernameThree = UtilIT.getUsernameFromResponse(createUserThree); + Response getUnavailableForDatasetBadSearchTerm = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiToken, "xxQQQ-Batman"); + getUnavailableForDatasetBadSearchTerm.prettyPrint(); + getUnavailableForDatasetBadSearchTerm.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", equalTo(0)); + Response getUnavailableForDataset = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiToken, dataverseAliasUnavailableForLinking); getUnavailableForDataset.prettyPrint(); getUnavailableForDataset.then().assertThat() From cadf11dd75c6c5e8389d64508bf659629f6c5d83 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 11 Sep 2025 14:04:18 -0400 Subject: [PATCH 193/634] #11710 Code Cleanup --- .../harvard/iq/dataverse/api/Dataverses.java | 33 ++++++------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 0abfed3d4ba..36e92f0f1a1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1783,7 +1783,6 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P try { DvObject dvObject = findDvoByIdAndTypeOrDie(dvIdtf, type, false); - List dataversesForLinkingSearch = new ArrayList(); User requestUser = (User) getRequestUser(crc); AuthenticatedUser authUser; @@ -1802,32 +1801,20 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P } else { searchParam = ""; } - dataversesForLinking = permissionService.findPermittedCollections(dvReq, authUser, Permission.LinkDataset, searchParam); - - List mergedWithSearch = new ArrayList<>(); - dataversesForLinking = dataverseService.removeUnlinkableDataverses(dataversesForLinking, dvObject); - - //Only do search lookup if search term is there. Otherwise just include the collections based on perms - /* - if (searchTerm != null && !searchTerm.isEmpty()) { - dataversesForLinkingSearch = dataverseService.filterDataversesByNamePattern(searchTerm); - if (!dataversesForLinkingSearch.isEmpty()) { - for (Dataverse dv : dataversesForLinking) { - if (dataversesForLinkingSearch.contains(dv)) { - mergedWithSearch.add(dv); - } - } - } + //Find Permitted Collections now takes a Search Term to filter down collections the user may link + Permission permToCheck; + if (dvObject instanceof Dataset) { + permToCheck = Permission.LinkDataset; } else { - //search term empty then add all based on perms - mergedWithSearch.addAll(dataversesForLinking); + permToCheck = Permission.LinkDataverse; } - */ - mergedWithSearch.addAll(dataversesForLinking); + + dataversesForLinking = permissionService.findPermittedCollections(dvReq, authUser, permToCheck, searchParam); + dataversesForLinking = dataverseService.removeUnlinkableDataverses(dataversesForLinking, dvObject); JsonArrayBuilder dvBuilder = Json.createArrayBuilder(); - if (!mergedWithSearch.isEmpty()) { - for (Dataverse dv : mergedWithSearch) { + if (!dataversesForLinking.isEmpty()) { + for (Dataverse dv : dataversesForLinking) { dvBuilder.add(json(dv, true)); } } From d0c656e759a61393ffb9366365712ddc18d5ac8a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 11 Sep 2025 14:46:51 -0400 Subject: [PATCH 194/634] more edits about S3 --- .../source/developers/big-data-support.rst | 87 +++++++++++++++++-- 1 file changed, 80 insertions(+), 7 deletions(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index ab8b92dff0f..76123827137 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -3,8 +3,8 @@ Scaling Dataverse with Data Size This guide is intended to help administrators configure Dataverse appropriately to handle the amount of data their installation needs to manage. -Scaling is a complex subject and there are many options available in Dataverse that can improve performance with larger scale data, but some of these -work differently than Dataverse's default configuration, potentially requiring user education, and can require additional expertise to manage. +Scaling is a complex subject: there are many options available in Dataverse that can improve performance with larger scale data, some of which +work differently than Dataverse's default configuration, potentially requiring user education, and some which can require additional expertise to manage. In general, there are three dimensions in which Dataverse can scale: @@ -12,15 +12,88 @@ In general, there are three dimensions in which Dataverse can scale: 2. **Number of Files per Dataset** 3. **Storage size of individual files and aggregate storage size** -This guide will primarily focus on the latter two dimensions. +This guide will primarily focus on the latter two dimensions as the main way to scale the number of datasets is to scale the database, server, and storage resources. .. contents:: |toctitle| :local: -Choosing the Right Approach ---------------------------- - -The table below provides guidance on which storage solution to choose based on your requirements: +Choosing the Right Storage +-------------------------- + +Dataverse provides several options that affect how files are transferred and stored which significantly impact the performance and scalability of your Dataverse installation. + +File Storage +~~~~~~~~~~~~ + +The default storage option in Dataverse is based on a local file system. When files are transferred to Dataverse, they are first stored in a +temporary location on the Dataverse server. Any zip files uploaded are unzipped to create multiple individual file entries. Once an upload is completed, +Dataverse copies the files to permanent storage. Dataverse also takes advantage of the file being local to inspect it's bytes to determine it +s MIME type, and, for tabular data, to 'ingest' it - extracting metadata about the variables used in the file and creating a tab-separated value (TSV) version of the file. + +Benefits: This option requires no external services and can potentially handle files into the gigabyte (GB) size range. FOr smaller institutions, +and in disciplines where datasets do not have more than a few hundred files and files are not too large, this can be the simplest option. +Further, unzipping of zip archives can be simpler for users than having to upload many individual files and was, at one time, the only way to +preserve file path names when uploading. + +Challenges: In general file storage is not a good option for larger data sizes - both in terms of file size and number of files. Contributing factors include: + +- Local file storage must be provisioned in advance based on anticipated demand. It can involve up-front costs (for a local disk), or, when procured from a cloud provider, is likely to be more expensive than object storage (see below) +- Because temporary storage is used, transfers will temporarily use several times as much space as the final transfer. Unzipping also increases the final storage size of a dataset. +- Because all uploads use the same temporary storage, temporary storage must be large enough to handle multiple users uploading data. +- Each file is uploaded as a single HTTP request, which can cause long transfer times which, in turn, can trigger timeout errors in Dataverse or any proxy or load balancer in front of Dataverse. +- Uploading many files at once can trigger any rate limiter (i.e. used to throttle use by AI) resulting in failures +- Because transfers (both uploads and downloads) are handled by the Dataverse server, they add to server processing load which can affect overall performance. + +Object Storage via S3 +~~~~~~~~~~~~~~~~~~~~~ + +A more scalable option for storage is to use an object store with transfers managed using the Simple Storage Service (S3) protocol. S3-compatible storage can +easily be bought (rented) from major cloud providers, but may also be available from institutional clouds. It is also possible to run open-source software to provide S3 storage +over a local file system (making it possible to enjoy the advantages discussed below while still leveraging local file storage). While Dataverse can be +configured to handle uploads and downloads as with file storage (with zip files being unzipped, but having many of the same challenges in terms of temporary storage and server load as discussed above) +it can also be configured to use 'direct' upload and download. In this configuration, the actual transfer of file bytes is from/to the user's local machine to/from +the S3 store. In this configuration, Dataverse does not attempt to unzip zip files and they are stored as a single file in the dataset. + +Benefits: S3 offers several advantages over file storage which : + +- Scalability: S3 is designed to handle large amounts of data. It can handle individual files up to several TB in size. +Because S3 supports breaking files into pieces, Dataverse can transfer a file in pieces (several in parallel, potentially thousands of pieces per file) making transfers faster +and more robust (a failure requires only resending the failed piece). It may also be the case that users will have a faster network connection to the S3 store +(e.g. in a commercial cloud or High Performance Computing center) than they do to the Dataverse server, reducing transfer time. +- High Availability: S3 provides redundancy beyond what is available with a single disk (valuable for preservation, potentially reducing the need to perform data integrity checks). + +Challenges: +- Cost: S3 offers a pricing model that allows you to pay for the storage and transfer of data based on current usage (versus long term demand) but commercial +providers charge more per TB than the equivalent cost of a local disk (though commercial S3 storage is cheaper than commercial file storage). +There can also be egress and other charges. Overall, S3 storage is generally more expensive than local file storage but cheaper than cloud file storage. +Running a local S3 storage or leveraging an institutional service can further reduce costs. +- S3 Storage without direct upload/download provides minimal benefits with Dataverse as files still pass through the server, files are still uploaded as a single HTTP/HTTPS stream, and temporary storage is still used. + +Other Considerations +^^^^^^^^^^^^^^^^^^^^ + +- While not having files unzipped can be confusing to users who are used to it from using Dataverse with file storage, there are ways to minimize the impact. +For example, Dataverse can be configured to use a 'Zip File Previewer' that allows users to see the contents of a zip file and even download individual files from within it. +For users who still want their data stored as individual files, Dataverse can be configured with the 'DVWebloader' which allows users to select an entire folder tree of files and +upload them, with their relative paths intact, to dataverse. (DVWebloader can only be used with S3/direct upload, but it is much more efficient in this case than using the +standard upload interface in Dataverse). +- Using direct upload stops Dataverse from inspecting the file bytes to determine the MIME type (with one exception - Stata files). Dataverse will still look at the file name and extension to determine the MIME type. +- To perform the 'ingest' processing, Dataverse currently has to copy the file to local storage, somewhat negating the benefit of sending data directly to S3. To manage larger files, one can set a per-store +Ingest size limit (which can be 0 bytes) to stop ingest or limit it to smaller files. +- Dataverse's mechanism for downloading a whole dataset or multiple selected files involves zipping those files together. Even When using S3 with direct upload/download, +the file bytes are transferred to the Dataverse server as part of the zipping process. There are ways to reduce the performance impact of this: + - There is a 'ZipDownloader' app that can be run separate from Dataverse to handle the zipping process. + - Dataverse has a :ZipDownloadLimit that can be used to limit the amount of data that can be zipped. If a dataset is larger than this limit, Dataverse will only add some of the files to the zip and list others in the included manifest file. + - There are tools such as the Dataverse Dataset Downloader (https://github.com/gdcc/dataverse-recipes/tree/main/shell/download#dataverse-dataset-downloader) that can be used to download all off the files individually. This avoids sending any of the files to/through the Dataverse server when S3 direct download is enabled. +- Dataverse leverages S3 features that are not implemented by all servers and has several configuration options geared towards handling variations between servers. Site admins should be sure to test with their preferred S3 implementation. + +Remote Storage +~~~~~~~~~~~~~~ + + + + +The table below provides basic guidance on Dataverse's which storage solution to choose based on your requirements: +------------------------+---------------------------+---------------------------+---------------------------+ | **Solution** | **File Size Range** | **File Volume Support** | **Key Benefits** | From d45d0b984e69711061a3d9b83977b77ad6eed6bc Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 11 Sep 2025 14:50:32 -0400 Subject: [PATCH 195/634] #11710 code cleanup --- .../java/edu/harvard/iq/dataverse/api/AbstractApiBean.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 6b5c4bcf0b2..0ce84844289 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -615,11 +615,12 @@ protected DvObject findDvo(@NotNull final String id) throws WrappedResponse { * * @param dvIdtf * @param type + * @param testForReleased * @return DvObject if type matches or throw exception * @throws WrappedResponse */ @NotNull - protected DvObject findDvoByIdAndTypeOrDie(@NotNull final String dvIdtf, String type, Boolean testForReleased) throws WrappedResponse { + protected DvObject findDvoByIdAndTypeOrDie(@NotNull final String dvIdtf, String type, boolean testForReleased) throws WrappedResponse { try { DataverseFeaturedItem.TYPES dvType = DataverseFeaturedItem.getDvType(type); DvObject dvObject = null; From 8d7c44140327f44d3939103bf00075f3ce54935d Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 11 Sep 2025 15:54:11 -0400 Subject: [PATCH 196/634] #11710 implement command --- .../harvard/iq/dataverse/api/Dataverses.java | 37 ++--------- .../impl/GetLinkingDataverseListCommand.java | 66 +++++++++++++++++++ .../iq/dataverse/api/DataversesIT.java | 2 +- 3 files changed, 74 insertions(+), 31 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 36e92f0f1a1..b65c53b0ded 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1783,35 +1783,12 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P try { DvObject dvObject = findDvoByIdAndTypeOrDie(dvIdtf, type, false); - - User requestUser = (User) getRequestUser(crc); - AuthenticatedUser authUser; - if (!requestUser.isAuthenticated()) { - throw new WrappedResponse(error(Response.Status.BAD_REQUEST, - BundleUtil.getStringFromBundle("dataverse.link.user"))); - } else { - authUser = (AuthenticatedUser) requestUser; - } - - DataverseRequest dvReq = new DataverseRequest(requestUser, (IpAddress) null); - List dataversesForLinking; - String searchParam; - if(searchTerm != null){ - searchParam = searchTerm; - } else { - searchParam = ""; - } - //Find Permitted Collections now takes a Search Term to filter down collections the user may link - Permission permToCheck; - if (dvObject instanceof Dataset) { - permToCheck = Permission.LinkDataset; - } else { - permToCheck = Permission.LinkDataverse; - } + List dataversesForLinking = execCommand(new GetLinkingDataverseListCommand( + createDataverseRequest(getRequestUser(crc)), + dvObject, + searchTerm + )); - dataversesForLinking = permissionService.findPermittedCollections(dvReq, authUser, permToCheck, searchParam); - dataversesForLinking = dataverseService.removeUnlinkableDataverses(dataversesForLinking, dvObject); - JsonArrayBuilder dvBuilder = Json.createArrayBuilder(); if (!dataversesForLinking.isEmpty()) { for (Dataverse dv : dataversesForLinking) { @@ -1821,8 +1798,8 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P return ok(dvBuilder); } catch (WrappedResponse wr) { return wr.getResponse(); - } - + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java new file mode 100644 index 00000000000..2710940c813 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java @@ -0,0 +1,66 @@ + +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.util.BundleUtil; +import java.util.List; + +/** + * + * @author stephenkraffmiller + */ +@RequiredPermissions({}) +public class GetLinkingDataverseListCommand extends AbstractCommand> { + + private final String searchTerm; + private final DvObject dvObject; + + public GetLinkingDataverseListCommand(DataverseRequest aRequest, DvObject dvObject, String searchTerm) { + super(aRequest, dvObject); + this.searchTerm = searchTerm; + this.dvObject = dvObject; + } + + @Override + public List execute(CommandContext ctxt) throws CommandException { + + User requestUser = (User) getRequest().getUser(); + AuthenticatedUser authUser; + if (!requestUser.isAuthenticated()) { + throw new IllegalCommandException(BundleUtil.getStringFromBundle("dataverse.link.user"), this); + } else { + authUser = (AuthenticatedUser) requestUser; + } + List dataversesForLinking; + String searchParam; + if (searchTerm != null) { + searchParam = searchTerm; + } else { + searchParam = ""; + } + + //Find Permitted Collections now takes a Search Term to filter down collections the user may link + Permission permToCheck; + if (dvObject instanceof Dataset) { + permToCheck = Permission.LinkDataset; + } else { + permToCheck = Permission.LinkDataverse; + } + + dataversesForLinking = ctxt.permissions().findPermittedCollections(getRequest(), authUser, permToCheck, searchParam); + return ctxt.dataverses().removeUnlinkableDataverses(dataversesForLinking, dvObject); + + } + +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index e849a32a513..c72a32a92f2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -802,7 +802,7 @@ public void testGetLinkableDataverses(){ Response getGuestUnavailableForDataset = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, "", dataverseAliasUnavailableForLinking); getGuestUnavailableForDataset.prettyPrint(); getGuestUnavailableForDataset.then().assertThat() - .statusCode(BAD_REQUEST.getStatusCode()); + .statusCode(FORBIDDEN.getStatusCode()); Response getUnavailableForDataverse = UtilIT.getLinkableDataverses("dataverse", dataverseAlias, apiToken, dataverseAliasUnavailableForLinking); getUnavailableForDataverse.prettyPrint(); From 3943cb201d9c262f616acb611caf2c03f530b7c4 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 11 Sep 2025 15:57:14 -0400 Subject: [PATCH 197/634] #11710 remove unused comment --- src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index b65c53b0ded..c2cb22bc195 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1778,7 +1778,6 @@ public Response linkDataverse(@Context ContainerRequestContext crc, @PathParam(" @Produces(MediaType.APPLICATION_JSON) @Path("{identifier}/{type}/linkingDataverses") public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @QueryParam("searchTerm") String searchTerm, @PathParam("type") String type) { - //first determine what you are linking based on identifier and type try { @@ -1799,7 +1798,6 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P } catch (WrappedResponse wr) { return wr.getResponse(); } - } From 46f170b120e1870f0c4d483a7800cf34b381b4cd Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 11 Sep 2025 17:10:43 -0400 Subject: [PATCH 198/634] remote, globus info --- .../source/developers/big-data-support.rst | 108 +++++++++++++++--- 1 file changed, 90 insertions(+), 18 deletions(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 76123827137..cf156bd12c8 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -62,7 +62,8 @@ and more robust (a failure requires only resending the failed piece). It may als (e.g. in a commercial cloud or High Performance Computing center) than they do to the Dataverse server, reducing transfer time. - High Availability: S3 provides redundancy beyond what is available with a single disk (valuable for preservation, potentially reducing the need to perform data integrity checks). -Challenges: +Challenges: + - Cost: S3 offers a pricing model that allows you to pay for the storage and transfer of data based on current usage (versus long term demand) but commercial providers charge more per TB than the equivalent cost of a local disk (though commercial S3 storage is cheaper than commercial file storage). There can also be egress and other charges. Overall, S3 storage is generally more expensive than local file storage but cheaper than cloud file storage. @@ -90,25 +91,98 @@ the file bytes are transferred to the Dataverse server as part of the zipping pr Remote Storage ~~~~~~~~~~~~~~ +Note: Remote Storage is still experimental: feedback is welcome! See :ref:`support`. + +For very large, and/or very sensitive data, it may not make sense to transfer or copy files to Dataverse at all. The ``remote`` store type in the Dataverse software support these use cases. +It allows Dataverse to store a URL reference for the file rather than transferring the file bytes to a store managed directly by Dataverse. +In the most basic configuration a site administrator configures the base URL for the store, e.g. "https://thirdpartystorage.edu/long-term-storage/" and users can then create files referencing +any URL starting with that base, e.g. "https://thirdpartystorage.edu/long-term-storage/my_file.txt". If the remote site is a public web server, the remote store in Dataverse should be configured to be 'public' which will +disable the ability to restrict or embargo files (as they are public on the remote site and Dataverse cannot block access.) Conversely, Dataverse can be configured to sign requests to the remote server and, +and, which the remote server, if it is capable of validating them, can use to reject requests not approved by Dataverse. In this configuration, users can restrict and embargo files and Dataverse and the remote server will cooperate to +manage access control. Another alternative, with a more advanced remote store, would be, instead of using URLs that directly enable download of the file, to use URLs that point to +a landing page at the remote server that may require the user to login, or go through some other authentication/validation process before being able to access the file. + +Dataverse considers remote storage to be read-only, or, in cases where the remote service does not provide a way for Dataverse to download the file bytes ((due to access control or because the URL refers to a landing page), inacessible. +Depending on whether Dataverse can access the bytes of the file, functionality such as ingest and integrity checking may or may not be possible. If the file bytes are not accessible +the remote store in Dataverse should be configured to disable operations that attempt to access the file (see files-not-accessible-by-dataverse). +If the file bytes are accessible, Dataverse can still support features such as ingest and thumbnail creation, as well as local storage of other files and auxilliary files. These are handled by configuring a 'base' store with the remote store +that is used for these purposes. (This means that while the specified files remain on the remote store, other files in the dataset, and potentially the ingested TSV format of a remote file would be managed by Dataverse in some other store. If ingest is not desired, the ingest size limit for the store can be set to 0 bytes). + +Benefits: + +- This is a relatively simple way to off-load the management of large and/or sensitive data files to other organizations while still providing Dataverse's overall capabilities for dataset curation and publication to users. +- If the store has been configured with a remote-store-name or remote-store-url, the dataset file table will include this information for remote files. These provide a visual indicator that the files are not managed directly by Dataverse and are stored/managed by a remote trusted store. + +Challenges: + +- Currently, remote files can only be added via the API. (This may be addressed in future versions). +- As Dataverse is relying on the remote service to main the integrity and availability of the files, it is likely that the Dataverse site admin will want to have a formal agreement with the remote service +operator about their policies. +- Site admins need to consider carefully how to configure file size limits, ingest size limits, etc. on the remote store and it's base store, and whether the remote store is public-only, and whether it files there can be read by Dataverse to assure the +requirements of a specific use case(s) are addressed. +- The current remote store implementation will not prevent you from providing a relative URL that results in a 404 when resolved. (I.e. if you make a typo). You should check to make sure the file exists at the location you specify - by trying to download in Dataverse, by checking to see that Dataverse was able to get the file size (which it does by doing a HEAD call to that location), or just manually trying the URL in your browser. +- For large files, direct-download should always be used with a remote store. (Otherwise the Dataverse will be involved in the download.) + + +Globus Transfer +~~~~~~~~~~~~~~~ + +Note: Globus Transfer is still experimental: feedback is welcome! See :ref:`support`. + +`Globus `_ provides file transfer service that is widely used for the largest datasets (in terms of both file size and number of files). It provides: + +- robust file transfer capable of handling delays (e.g. due to the time it takes to mount tapes) and restarting after network or endpoint failures +- parallel file transfers, potentially between clusters of computers on both ends +- third-party transfer, which enables a user working with their desktop browser to initiate transfer of files from one remote endpoint (i.e. on a local high-performance computing cluster) to/from another (e.g. one associated with a Dataverse store) + +Dataverse can be configured to support Globus transfers in multiple ways: + +- A Dataverse-managed Globus File Endpoint: Dataverse controls user access to the endpoint, access is only via Globus +- A Dataverse-managed Globus S3 Endpoint: Dataverse controls user access to the endpoint, access is available via S3 and via Globus +- A Globus Endpoint treated as Remote Storage: Dataverse references files on a Globus endpoint managed by a third party + +Each of these options has its own advantages and disadvantages: +Benefits: + +- Globus scales to higher data volumes than any other option. Users working with large data are often familiar with Globus and are interested in transferring data to/from computational clusters rather than their local machine +- Globus transfers can be initiated by choosing the Globus option in the dataset upload panel. Analogously, "Globus Transfer" is one of the download options in the "Access Dataset" menu. +- For the non-S3 options, Dataverse support having a base store (e.g. a local file system or an S3-based store), which can be used internally by Dataverse (e.g. for thumbnails, etc.) and can allow users to upload smaller files (e.g. Readmes, documentation) that might not be suited to a given Globus endpoint (e.g. a tape store) + +Challenges: +- Globus is complex to manage and Dataverse installations will need to develop Globus expertise or partner with another organization (i.e. a institutional high-performance computing center) to manage Globus endpoints. +- For users not familiar with Globus, managing transfers can be confusing. Globus does provide a free 'Globus Personal Connect' service which can be run on any machine to allow transfers to/from it. +- Globus transfers are not enabled at dataset-creation time. Once the draft version is created, users can initiate Globus transfers to upload files from remote endpoints. +- For Dataverse managed endpoints, a a community-developed `dataverse-globus `_ app must be installed and configured in the Dataverse instance. +This app manages granting and revoking access for users to upload/download files from Dataverse and handles the translation between Dataverse's internal file naming/organization to that see by the user. +- Due to differences between Dataverse's and Globus's access control models, Dataverse cannot enforce per-file-access restrictions - restriction can only be done today at the level of providing access to all files in a dataset. +Globus stores can be defined as public to disable Dataverse's ability to restrict and embargo files in that store. If the store is configured to support restriction and embargo, +Dataverse and it's Dataverse-Globus app will limit users to downloading only the files they have been granted access to, but a technically knowledgeable user could access other files in the same dataset if they are give access to one. +(Data depositors would need to be aware of this limitation and could be guided to restrict all files/only grant access to all dataset files as a work-around). +- Dataverse-managed endpoints must be Globus 'guest collections' hosted on either a file-system-based endpoint or an S3-based endpoint (the latter requires use of the Globus +S3 connector which requires a paid Globus subscription at the host institution). In either case, Dataverse is configured with the Globus credentials of a user account that can manage the endpoint. +Users will need their own Globus account, which can be obtained via their institution or directly from Globus (at no cost). +- With the file-system endpoint, Dataverse does not currently have access to the file contents. Thus, functionality related to ingest, previews, fixity hash validation, etc. are not available. (Using the S3-based endpoint, Dataverse has access via S3 and all functionality normally associated with direct uploads to S3 is available.) +- For the reference use case, Dataverse must be configured with a list of allowed endpoint/base paths from which files may be referenced. In this case, since Dataverse is not accessing the remote endpoint itself, it does not need Globus credentials. +Users will also need a Globus account in this case, and the remote endpoint must be configured to allow them access (i.e. be publicly readable, or potentially involving some out-of-band mechanism to request access (that could be described in the dataset's Terms of Use and Access). +- While Globus itself can handle many (millions of) files of any size, Dataverse cannot handle more than thousands of files per dataset (at best) and some Globus endpoints may have limits on file sizes - both maximums and minimums (e.g. for tape storage where small files are inefficient). +Users will need to be made aware of these limitations and the possibilities for managing them (e.g. by aggregating multiple files in a single larger file). + +More details of the setup required to enable Globus is described in the `Community Dataverse-Globus Setup and Configuration document `_ and the references therein. + +As described in that document, + +An overview of the control and data transfer interactions between components was presented at the 2022 Dataverse Community Meeting and can be viewed in the `Integrations and Tools Session Video `_ around the 1 hr 28 min mark. + +See also :ref:`Globus settings <:GlobusSettings>`. + +An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. Due to its experimental nature it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag (see :ref:`feature-flags`) and the JVM option :ref:`dataverse.files.globus-monitoring-server`. + + + -The table below provides basic guidance on Dataverse's which storage solution to choose based on your requirements: -+------------------------+---------------------------+---------------------------+---------------------------+ -| **Solution** | **File Size Range** | **File Volume Support** | **Key Benefits** | -+========================+===========================+===========================+===========================+ -| Default Storage | Up to ~2GB | Up to thousands | Simple, no configuration | -+------------------------+---------------------------+---------------------------+---------------------------+ -| S3 Direct Upload | Up to several TB | Up to tens of thousands | Simple configuration, | -| | | | familiar user interface | -+------------------------+---------------------------+---------------------------+---------------------------+ -| Remote Storage | Unlimited | Unlimited | No file transfer needed, | -| | | | files remain in place | -+------------------------+---------------------------+---------------------------+---------------------------+ -| Globus Transfer | Unlimited | Millions+ | Robust transfer, | -| | | | resumable, high-speed | -+------------------------+---------------------------+---------------------------+---------------------------+ Handling Large Individual Files ------------------------------- @@ -263,8 +337,6 @@ To configure the options mentioned above, an administrator must set two JVM opti Globus File Transfer ~~~~~~~~~~~~~~~~~~~ -**Best for:** Very large files and high-performance computing environments - Note: Globus file transfer is still experimental but feedback is welcome! See :ref:`support`. Users can transfer files via `Globus `_ into and out of datasets, or reference files on a remote Globus endpoint, when their Dataverse installation is configured to use a Globus accessible store(s) From c7c7e7015b0d0871c7634e5250ef9c5fa6099f78 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 11 Sep 2025 17:20:06 -0400 Subject: [PATCH 199/634] Apply suggestions from code review Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 4 ++-- .../edu/harvard/iq/dataverse/DataverseRoleServiceBean.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 083d72d01cf..5d5e057345d 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1508,7 +1508,7 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept: text/csv" "https://demo.dataverse.org/api/dataverses/3/assignments/history" -The CSV response has column headers mirroring the json entries. They are internationalized (when internationalization is configured). +The CSV response has column headers mirroring the JSON entries. They are internationalized (when internationalization is configured). Note: This feature requires the "role-assignment-history" feature flag to be enabled (see :ref:`feature-flags`). @@ -4199,7 +4199,7 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept: text/csv" "https://demo.dataverse.org/api/datasets/3/assignments/history" -The CSV response has column headers mirroring the json entries. They are internationalized (when internationalization is configured). +The CSV response has column headers mirroring the JSON entries. They are internationalized (when internationalization is configured). Note: This feature requires the "role-assignment-history" feature flag to be enabled (see :ref:`feature-flags`). diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java index a341735c0e7..f16726a8fbe 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java @@ -469,7 +469,7 @@ public static class RoleAssignmentHistoryConsolidatedEntry { private Date assignedAt; private String revokedBy; private Date revokedAt; - private List definitionPointIds; // New field + private List definitionPointIds; public RoleAssignmentHistoryConsolidatedEntry(String assigneeIdentifier, String roleName, Long definitionPointId) { this.roleName = roleName; From 62f976e31876afc4805547dad4b722d9344f0a85 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 11 Sep 2025 17:36:44 -0400 Subject: [PATCH 200/634] updates per review --- doc/release-notes/11612-RAHistory.md | 2 +- .../edu/harvard/iq/dataverse/ManageFilePermissionsPage.java | 3 ++- .../java/edu/harvard/iq/dataverse/ManagePermissionsPage.java | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/release-notes/11612-RAHistory.md b/doc/release-notes/11612-RAHistory.md index b61fc9d0406..82df91c3913 100644 --- a/doc/release-notes/11612-RAHistory.md +++ b/doc/release-notes/11612-RAHistory.md @@ -4,7 +4,7 @@ Dataverse can now track the history of role assignments, allowing administrators ## Key components of this feature: -- **Feature Flag**: The functionality can be enabled/disabled via the `ROLE_ASSIGNMENT_HISTORY` feature flag +- **Feature Flag**: The functionality can be enabled/disabled via the `ROLE_ASSIGNMENT_HISTORY` feature flag (default is `off`) - **UI Integration**: New history panels on permission management pages showing the complete history of role assignments/revocations - **CSV Export**: Administrators can download the role assignment history for a given collection or dataset (or files in a dataset) as a CSV file directly from the new panels - **API Access**: New API endpoints provide access to role assignment history in both JSON and CSV formats: diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java index 0c49346fd80..a23b41cb1d1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java @@ -584,7 +584,8 @@ public void setRenderFileMessages(boolean renderFileMessages) { this.renderFileMessages = renderFileMessages; } - public String getsignedUrlForRAHistoryCsv() { + public String getSignedUrlForRAHistoryCsv() { + //Including /v1 is required for the signature to validate String apiPath = "/api/v1/datasets/" + dataset.getId() + "/files/permissions/history"; try { diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java index 1f980c877ba..f2a2dfb5df7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java @@ -696,9 +696,9 @@ public void setRenderRoleMessages(Boolean renderRoleMessages) { this.renderRoleMessages = renderRoleMessages; } - public String getsignedUrlForRAHistoryCsv() { + public String getSignedUrlForRAHistoryCsv() { String apiPath; - + //Including /v1 in these urls is required for the signature to validate if (dvObject instanceof Dataverse dv) { // For Dataverses, use the dataverses API endpoint with the alias apiPath = "/api/v1/dataverses/" + dv.getAlias() + "/permissions/history"; From 70c7430fb20b56d6cf47ba5c06fafd9c6fd4067f Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 12 Sep 2025 14:45:22 -0400 Subject: [PATCH 201/634] #11710 simplify logic --- .../iq/dataverse/util/json/JsonPrinter.java | 79 ++++++++++--------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 9c7c519c1ef..46aa2dfff20 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -310,48 +310,49 @@ public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean re .add("alias", dv.getAlias()) .add("name", dv.getName()); //minimal refers to only returning the id alias and name for - //use in selecting collections available for linking - if (!minimal) { - bld.add("affiliation", dv.getAffiliation()); - if (!hideEmail) { - bld.add("dataverseContacts", JsonPrinter.json(dv.getDataverseContacts())); - } - if (returnOwners) { - bld.add("isPartOf", getOwnersFromDvObject(dv)); - } - bld.add("permissionRoot", dv.isPermissionRoot()) - .add("description", dv.getDescription()) - .add("dataverseType", dv.getDataverseType().name()) - .add("isMetadataBlockRoot", dv.isMetadataBlockRoot()) - .add("isFacetRoot", dv.isFacetRoot()); - if (dv.getOwner() != null) { - bld.add("ownerId", dv.getOwner().getId()); - } - if (dv.getCreateDate() != null) { - bld.add("creationDate", Util.getDateTimeFormat().format(dv.getCreateDate())); - } - if (dv.getDataverseTheme() != null) { - bld.add("theme", JsonPrinter.json(dv.getDataverseTheme())); - } - if (dv.getStorageDriverId() != null) { - bld.add("storageDriverLabel", DataAccess.getStorageDriverLabelFor(dv.getStorageDriverId())); - } - if (dv.getFilePIDsEnabled() != null) { - bld.add("filePIDsEnabled", dv.getFilePIDsEnabled()); - } - bld.add("effectiveRequiresFilesToPublishDataset", dv.getEffectiveRequiresFilesToPublishDataset()); - bld.add("isReleased", dv.isReleased()); + //used in selecting collections available for linking + if (minimal) { + return bld; + } + bld.add("affiliation", dv.getAffiliation()); + if (!hideEmail) { + bld.add("dataverseContacts", JsonPrinter.json(dv.getDataverseContacts())); + } + if (returnOwners) { + bld.add("isPartOf", getOwnersFromDvObject(dv)); + } + bld.add("permissionRoot", dv.isPermissionRoot()) + .add("description", dv.getDescription()) + .add("dataverseType", dv.getDataverseType().name()) + .add("isMetadataBlockRoot", dv.isMetadataBlockRoot()) + .add("isFacetRoot", dv.isFacetRoot()); + if (dv.getOwner() != null) { + bld.add("ownerId", dv.getOwner().getId()); + } + if (dv.getCreateDate() != null) { + bld.add("creationDate", Util.getDateTimeFormat().format(dv.getCreateDate())); + } + if (dv.getDataverseTheme() != null) { + bld.add("theme", JsonPrinter.json(dv.getDataverseTheme())); + } + if (dv.getStorageDriverId() != null) { + bld.add("storageDriverLabel", DataAccess.getStorageDriverLabelFor(dv.getStorageDriverId())); + } + if (dv.getFilePIDsEnabled() != null) { + bld.add("filePIDsEnabled", dv.getFilePIDsEnabled()); + } + bld.add("effectiveRequiresFilesToPublishDataset", dv.getEffectiveRequiresFilesToPublishDataset()); + bld.add("isReleased", dv.isReleased()); - List inputLevels = dv.getDataverseFieldTypeInputLevels(); - if (!inputLevels.isEmpty()) { - bld.add("inputLevels", JsonPrinter.jsonDataverseFieldTypeInputLevels(inputLevels)); - } + List inputLevels = dv.getDataverseFieldTypeInputLevels(); + if (!inputLevels.isEmpty()) { + bld.add("inputLevels", JsonPrinter.jsonDataverseFieldTypeInputLevels(inputLevels)); + } - if (childCount != null) { - bld.add("childCount", childCount); - } - addDatasetFileCountLimit(dv, bld); + if (childCount != null) { + bld.add("childCount", childCount); } + addDatasetFileCountLimit(dv, bld); return bld; } From 47d7e3dd506234ab7bb133dc0358bd340ac2d4a7 Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 15 Sep 2025 11:36:19 +0100 Subject: [PATCH 202/634] Added: GetLinkingDataverseListCommand unit test --- .../GetLinkingDataverseListCommandTest.java | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 src/test/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommandTest.java diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommandTest.java new file mode 100644 index 00000000000..50636474be8 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommandTest.java @@ -0,0 +1,161 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.util.BundleUtil; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ExtendWith(MockitoExtension.class) +public class GetLinkingDataverseListCommandTest { + + @Mock + private DataverseRequest dataverseRequest; + @Mock + private CommandContext commandContext; + @Mock + private PermissionServiceBean permissionService; + @Mock + private DataverseServiceBean dataverseService; + @Mock + private AuthenticatedUser authenticatedUser; + + @BeforeEach + public void setUp() { + Mockito.when(dataverseRequest.getUser()).thenReturn(authenticatedUser); + } + + @Test + public void execute_shouldThrowException_whenUserIsNotAuthenticated() { + // Arrange + Mockito.when(authenticatedUser.isAuthenticated()).thenReturn(false); + DvObject dvObject = new Dataset(); + GetLinkingDataverseListCommand sut = new GetLinkingDataverseListCommand(dataverseRequest, dvObject, ""); + + // Act & Assert + assertThrows(IllegalCommandException.class, () -> { + sut.execute(commandContext); + }, BundleUtil.getStringFromBundle("dataverse.link.user")); + } + + @Test + public void execute_shouldReturnPermittedList_forLinkingDataset() throws CommandException { + // Arrange + String searchTerm = "test"; + Dataset datasetToLink = new Dataset(); + List permittedList = Arrays.asList(new Dataverse(), new Dataverse()); + + Mockito.when(commandContext.permissions()).thenReturn(permissionService); + Mockito.when(commandContext.dataverses()).thenReturn(dataverseService); + Mockito.when(authenticatedUser.isAuthenticated()).thenReturn(true); + Mockito.when(permissionService.findPermittedCollections(dataverseRequest, authenticatedUser, Permission.LinkDataset, searchTerm)) + .thenReturn(permittedList); + Mockito.when(dataverseService.removeUnlinkableDataverses(permittedList, datasetToLink)) + .thenReturn(permittedList); // Assume none are removed for this test + + GetLinkingDataverseListCommand sut = new GetLinkingDataverseListCommand(dataverseRequest, datasetToLink, searchTerm); + + // Act + List result = sut.execute(commandContext); + + // Assert + assertEquals(2, result.size()); + assertEquals(permittedList, result); + Mockito.verify(permissionService).findPermittedCollections(dataverseRequest, authenticatedUser, Permission.LinkDataset, searchTerm); + Mockito.verify(dataverseService).removeUnlinkableDataverses(permittedList, datasetToLink); + } + + @Test + public void execute_shouldReturnPermittedList_forLinkingDataverse() throws CommandException { + // Arrange + String searchTerm = "test"; + Dataverse dataverseToLink = new Dataverse(); + List permittedList = Collections.singletonList(new Dataverse()); + + Mockito.when(commandContext.permissions()).thenReturn(permissionService); + Mockito.when(commandContext.dataverses()).thenReturn(dataverseService); + Mockito.when(authenticatedUser.isAuthenticated()).thenReturn(true); + Mockito.when(permissionService.findPermittedCollections(dataverseRequest, authenticatedUser, Permission.LinkDataverse, searchTerm)) + .thenReturn(permittedList); + Mockito.when(dataverseService.removeUnlinkableDataverses(permittedList, dataverseToLink)) + .thenReturn(permittedList); + + GetLinkingDataverseListCommand sut = new GetLinkingDataverseListCommand(dataverseRequest, dataverseToLink, searchTerm); + + // Act + List result = sut.execute(commandContext); + + // Assert + assertEquals(1, result.size()); + assertEquals(permittedList, result); + Mockito.verify(permissionService).findPermittedCollections(dataverseRequest, authenticatedUser, Permission.LinkDataverse, searchTerm); + Mockito.verify(dataverseService).removeUnlinkableDataverses(permittedList, dataverseToLink); + } + + @Test + public void execute_shouldUseEmptyString_whenSearchTermIsNull() throws CommandException { + // Arrange + String searchTerm = null; + Dataset datasetToLink = new Dataset(); + List expectedList = Collections.emptyList(); + + Mockito.when(commandContext.permissions()).thenReturn(permissionService); + Mockito.when(commandContext.dataverses()).thenReturn(dataverseService); + Mockito.when(authenticatedUser.isAuthenticated()).thenReturn(true); + // Note: verify that an empty string "" is passed instead of null + Mockito.when(permissionService.findPermittedCollections(dataverseRequest, authenticatedUser, Permission.LinkDataset, "")) + .thenReturn(expectedList); + Mockito.when(dataverseService.removeUnlinkableDataverses(expectedList, datasetToLink)) + .thenReturn(expectedList); + + GetLinkingDataverseListCommand sut = new GetLinkingDataverseListCommand(dataverseRequest, datasetToLink, searchTerm); + + // Act + List result = sut.execute(commandContext); + + // Assert + assertTrue(result.isEmpty()); + Mockito.verify(permissionService).findPermittedCollections(dataverseRequest, authenticatedUser, Permission.LinkDataset, ""); + } + + @Test + public void execute_shouldReturnEmptyList_whenNoPermittedCollectionsFound() throws CommandException { + // Arrange + String searchTerm = "nonexistent"; + Dataset datasetToLink = new Dataset(); + List emptyList = Collections.emptyList(); + + Mockito.when(commandContext.permissions()).thenReturn(permissionService); + Mockito.when(commandContext.dataverses()).thenReturn(dataverseService); + Mockito.when(authenticatedUser.isAuthenticated()).thenReturn(true); + Mockito.when(permissionService.findPermittedCollections(dataverseRequest, authenticatedUser, Permission.LinkDataset, searchTerm)) + .thenReturn(emptyList); + Mockito.when(dataverseService.removeUnlinkableDataverses(emptyList, datasetToLink)) + .thenReturn(emptyList); + + GetLinkingDataverseListCommand sut = new GetLinkingDataverseListCommand(dataverseRequest, datasetToLink, searchTerm); + + // Act + List result = sut.execute(commandContext); + + // Assert + assertTrue(result.isEmpty()); + } +} From 72e78a5f0d6c18f638910abde802061fa7e9a517 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 16 Sep 2025 09:56:31 -0400 Subject: [PATCH 203/634] #11710 short circuit test for unlinkable --- .../engine/command/impl/GetLinkingDataverseListCommand.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java index 2710940c813..c47c6215640 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java @@ -59,6 +59,10 @@ public List execute(CommandContext ctxt) throws CommandException { } dataversesForLinking = ctxt.permissions().findPermittedCollections(getRequest(), authUser, permToCheck, searchParam); + //Don't bother with checking for already linked if there are none to be tested. + if(dataversesForLinking == null || dataversesForLinking.isEmpty()) { + return dataversesForLinking; + } return ctxt.dataverses().removeUnlinkableDataverses(dataversesForLinking, dvObject); } From 844ca2c9c37706851a06b20281abea434accac02 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 16 Sep 2025 10:23:37 -0400 Subject: [PATCH 204/634] #11710 comment out unused mocks --- .../impl/GetLinkingDataverseListCommandTest.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommandTest.java index 50636474be8..7f167370c63 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommandTest.java @@ -117,13 +117,14 @@ public void execute_shouldUseEmptyString_whenSearchTermIsNull() throws CommandEx List expectedList = Collections.emptyList(); Mockito.when(commandContext.permissions()).thenReturn(permissionService); - Mockito.when(commandContext.dataverses()).thenReturn(dataverseService); + // if there are none found then the remove unlinkable doesn't get run + // Mockito.when(commandContext.dataverses()).thenReturn(dataverseService); Mockito.when(authenticatedUser.isAuthenticated()).thenReturn(true); // Note: verify that an empty string "" is passed instead of null Mockito.when(permissionService.findPermittedCollections(dataverseRequest, authenticatedUser, Permission.LinkDataset, "")) .thenReturn(expectedList); - Mockito.when(dataverseService.removeUnlinkableDataverses(expectedList, datasetToLink)) - .thenReturn(expectedList); + // if there are none found then the remove unlinkable doesn't get run + // Mockito.when(dataverseService.removeUnlinkableDataverses(expectedList, datasetToLink)).thenReturn(expectedList); GetLinkingDataverseListCommand sut = new GetLinkingDataverseListCommand(dataverseRequest, datasetToLink, searchTerm); @@ -143,12 +144,13 @@ public void execute_shouldReturnEmptyList_whenNoPermittedCollectionsFound() thro List emptyList = Collections.emptyList(); Mockito.when(commandContext.permissions()).thenReturn(permissionService); - Mockito.when(commandContext.dataverses()).thenReturn(dataverseService); + // if there are none found then the remove unlinkable doesn't get run + // Mockito.when(commandContext.dataverses()).thenReturn(dataverseService); Mockito.when(authenticatedUser.isAuthenticated()).thenReturn(true); Mockito.when(permissionService.findPermittedCollections(dataverseRequest, authenticatedUser, Permission.LinkDataset, searchTerm)) .thenReturn(emptyList); - Mockito.when(dataverseService.removeUnlinkableDataverses(emptyList, datasetToLink)) - .thenReturn(emptyList); + // if there are none found then the remove unlinkable doesn't get run + // Mockito.when(dataverseService.removeUnlinkableDataverses(emptyList, datasetToLink)).thenReturn(emptyList); GetLinkingDataverseListCommand sut = new GetLinkingDataverseListCommand(dataverseRequest, datasetToLink, searchTerm); From bd900fb4a1488c68285232056f78ce5ba6b7204e Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 16 Sep 2025 13:12:57 -0400 Subject: [PATCH 205/634] #11710 another null check --- src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index c2cb22bc195..f5816c36852 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1789,7 +1789,7 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P )); JsonArrayBuilder dvBuilder = Json.createArrayBuilder(); - if (!dataversesForLinking.isEmpty()) { + if (dataversesForLinking != null && !dataversesForLinking.isEmpty()) { for (Dataverse dv : dataversesForLinking) { dvBuilder.add(json(dv, true)); } From cfa64edc150ef2a29d64b59c0b634dc660873e08 Mon Sep 17 00:00:00 2001 From: Monu Kumari Date: Tue, 16 Sep 2025 23:55:31 +0530 Subject: [PATCH 206/634] =?UTF-8?q?docs:=20fix=20typo=20'orgization'=20?= =?UTF-8?q?=E2=86=92=20'organization'=20in=20Installation=20config=20(#118?= =?UTF-8?q?20)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/sphinx-guides/source/installation/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 14bf33c9482..7eec578566b 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3189,7 +3189,7 @@ dataverse.person-or-org.org-phrase-array Please note that this setting is experimental. The Schema.org metadata and OpenAIRE exports and the Schema.org metadata included in DatasetPages try to infer whether each entry in the various fields (e.g. Author, Contributor) is a Person or Organization. -If you have examples where an orgization name is being inferred to belong to a person, you can use this setting to force it to be recognized as an organization. +If you have examples where an organization name is being inferred to belong to a person, you can use this setting to force it to be recognized as an organization. The value is expected to be a comma-separated list of strings. Any name that contains one of the strings is assumed to be an organization. For example, "Project" is a word that is not otherwise associated with being an organization. Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_PERSON_OR_ORG_ORG_PHRASE_ARRAY``. From 8756f5c084135f0b8b7044594f93f25374b1c8cd Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 16 Sep 2025 14:50:40 -0400 Subject: [PATCH 207/634] #11710 fix super user query/run --- .../iq/dataverse/PermissionServiceBean.java | 39 +++++++++++++++++-- .../iq/dataverse/api/DataversesIT.java | 20 +++++++++- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index 1fb0d52e9fc..d492991bb62 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -96,7 +96,7 @@ public class PermissionServiceBean { DatasetVersionFilesServiceBean datasetVersionFilesServiceBean; private static final String LIST_ALL_DATAVERSES_SUPERUSER_HAS_PERMISSION = """ - SELECT id, name, alias FROM DATAVERSE + SELECT id, name, alias FROM DATAVERSE dv """; private static final String LIST_ALL_DATAVERSES_USER_HAS_PERMISSION = """ @@ -163,9 +163,17 @@ AND EXISTS (SELECT id FROM dataverserole WHERE dataverserole.id = roleassignment ) ) """; + + private static final String AND = """ + and + """; + + private static final String WHERE = """ + where + """; private static final String SEARCH_PARAMS = """ - and ((LOWER(dv.name) LIKE ? and ((SUBSTRING(LOWER(dv.name),0,(LENGTH(dv.name)-9)) LIKE ?) + ((LOWER(dv.name) LIKE ? and ((SUBSTRING(LOWER(dv.name),0,(LENGTH(dv.name)-9)) LIKE ?) or (SUBSTRING(LOWER(dv.name),0,(LENGTH(dv.name)-9)) LIKE ?))) or (LOWER(dv.name) NOT LIKE ? and ((LOWER(dv.name) LIKE ?) or (LOWER(dv.name) LIKE ?)))) @@ -975,6 +983,31 @@ public List findPermittedCollections(DataverseRequest request, Authen String sqlCode; if (user.isSuperuser()) { sqlCode = LIST_ALL_DATAVERSES_SUPERUSER_HAS_PERMISSION; + + if (searchTerm == null || searchTerm.isEmpty()) { + return em.createNativeQuery(sqlCode, Dataverse.class).getResultList(); + } else { + sqlCode = LIST_ALL_DATAVERSES_SUPERUSER_HAS_PERMISSION.concat(WHERE).concat(SEARCH_PARAMS); + + String pattern = searchTerm.toLowerCase(); + String pattern1 = pattern + "%"; + String pattern2 = "% " + pattern + "%"; + + // Adjust the queries for very short, 1 + if (pattern.length() == 1) { + pattern1 = pattern; + pattern2 = pattern + " %"; + } + Query query = em.createNativeQuery(sqlCode, Dataverse.class); + query.setParameter(1, "%dataverse"); + query.setParameter(2, pattern1); + query.setParameter(3, pattern2); + query.setParameter(4, "%dataverse"); + query.setParameter(5, pattern1); + query.setParameter(6, pattern2); + return query.getResultList(); + + } } else { if (searchTerm == null || searchTerm.isEmpty()) { sqlCode = LIST_ALL_DATAVERSES_USER_HAS_PERMISSION @@ -993,7 +1026,7 @@ public List findPermittedCollections(DataverseRequest request, Authen pattern2 = pattern + " %"; } - sqlCode = LIST_ALL_DATAVERSES_USER_HAS_PERMISSION.concat(SEARCH_PARAMS) + sqlCode = LIST_ALL_DATAVERSES_USER_HAS_PERMISSION.concat(AND).concat(SEARCH_PARAMS) .replace("@USERID", String.valueOf(user.getId())) .replace("@PERMISSIONBIT", String.valueOf(permissionBit)) .replace("@IPRANGESQL", ipRangeSQL); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index c72a32a92f2..bbe1291f90f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -45,6 +45,7 @@ import io.restassured.path.json.JsonPath; import org.hamcrest.CoreMatchers; import org.hamcrest.Matchers; +import static org.hamcrest.Matchers.greaterThan; public class DataversesIT { @@ -837,11 +838,28 @@ public void testGetLinkableDataverses(){ //set user api back to super user for cleanup UtilIT.setSuperuserStatus(username, Boolean.TRUE); - + + //Create dataverse that should be available for super user + Response createDataverseResponseAvailableForLinkingBySuperUser = UtilIT.createRandomDataverse(apiTokenTwo); + createDataverseResponseAvailableForLinkingBySuperUser.prettyPrint(); + String dataverseAliasAvailableForLinkingBySuperUser = UtilIT.getAliasFromResponse(createDataverseResponseAvailableForLinkingBySuperUser); + publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAliasAvailableForLinkingBySuperUser, apiToken); + publishDataverse.prettyPrint(); + assertEquals(200, publishDataverse.getStatusCode()); + + //First see that the super user gets dataverses to link to + getLinkableDataverses = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiToken, dataverseAliasAvailableForLinkingBySuperUser); + getLinkableDataverses.prettyPrint(); + getLinkableDataverses.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", equalTo(1)); // Clean up Response destroyDatasetResponse = UtilIT.destroyDataset(datasetId, apiToken); assertEquals(200, destroyDatasetResponse.getStatusCode()); + + Response deleteDataverseResponseSuper = UtilIT.deleteDataverse(dataverseAliasAvailableForLinkingBySuperUser, apiToken); + assertEquals(200, deleteDataverseResponseSuper.getStatusCode()); Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); assertEquals(200, deleteDataverseResponse.getStatusCode()); From be6ee1a991eda9b6bab998f63fc3143400119c4e Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 16 Sep 2025 13:57:08 -0400 Subject: [PATCH 208/634] break permission reindex into multiple transactions --- .../search/SolrIndexServiceBean.java | 192 ++++++++++-------- 1 file changed, 106 insertions(+), 86 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index 64679b05beb..01b9960e76b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -28,6 +28,8 @@ import jakarta.ejb.EJB; import jakarta.ejb.Stateless; +import jakarta.ejb.TransactionAttribute; +import jakarta.ejb.TransactionAttributeType; import jakarta.inject.Named; import jakarta.json.Json; import jakarta.json.JsonObjectBuilder; @@ -43,6 +45,9 @@ public class SolrIndexServiceBean { private static final Logger logger = Logger.getLogger(SolrIndexServiceBean.class.getCanonicalName()); + @EJB + private SolrIndexServiceBean self; // Self-injection to allow calling methods in new transactions (from other methods in this bean) + @EJB DvObjectServiceBean dvObjectService; @EJB @@ -317,7 +322,7 @@ private void persistToSolr(Collection docs) throws SolrServer /** * We use the database to determine direct children since there is no - * inheritance + * inheritance. This implementation uses smaller transactions to avoid memory issues. */ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) { @@ -325,114 +330,129 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) logger.log(Level.WARNING, "Cannot perform indexPermissionsOnSelfAndChildren with a definitionPoint null"); return null; } - int fileQueryMin= JvmSettings.MIN_FILES_TO_USE_PROXY.lookupOptional(Integer.class).orElse(Integer.MAX_VALUE); - List filesToReindexAsBatch = new ArrayList<>(); - /** - * @todo Re-indexing the definition point itself seems to be necessary - * for revoke but not necessarily grant. - */ - - // We don't create a Solr "primary/content" doc for the root dataverse - // so don't create a Solr "permission" doc either. - final int[] counter = {0}; + int fileQueryMin = JvmSettings.MIN_FILES_TO_USE_PROXY.lookupOptional(Integer.class).orElse(Integer.MAX_VALUE); + final int[] counter = { 0 }; int numObjects = 0; long globalStartTime = System.currentTimeMillis(); - if (definitionPoint.isInstanceofDataverse()) { - Dataverse selfDataverse = (Dataverse) definitionPoint; - if (!selfDataverse.equals(dataverseService.findRootDataverse())) { + + // Handle the definition point itself in its own transaction + if (definitionPoint instanceof Dataverse dataverse) { + // We don't create a Solr "primary/content" doc for the root dataverse + // so don't create a Solr "permission" doc either. + if (!dataverse.equals(dataverseService.findRootDataverse())) { indexPermissionsForOneDvObject(definitionPoint); numObjects++; } - List directChildDatasetsOfDvDefPoint = datasetService.findByOwnerId(selfDataverse.getId()); - for (Dataset dataset : directChildDatasetsOfDvDefPoint) { + + // Process datasets in batches + List datasetIds = datasetService.findIdsByOwnerId(dataverse.getId()); + int batchSize = 10; // Process 10 datasets per transaction + + for (int i = 0; i < datasetIds.size(); i += batchSize) { + int endIndex = Math.min(i + batchSize, datasetIds.size()); + List batchIds = datasetIds.subList(i, endIndex); + + // Process this batch of datasets in a new transaction + self.indexDatasetBatchInNewTransaction(batchIds, counter, fileQueryMin); + numObjects += batchIds.size(); + + logger.info("Processed batch " + (i / batchSize + 1) + " of " + + (int) Math.ceil(datasetIds.size() / (double) batchSize) + + " dataset batches for dataverse " + dataverse.getId()); + } + } else if (definitionPoint instanceof Dataset dataset) { + // For a single dataset, process it in its own transaction + indexPermissionsForOneDvObject(definitionPoint); + numObjects++; + + // Process the dataset's files in a new transaction + self.indexDatasetFilesInNewTransaction(dataset.getId(), counter, fileQueryMin); + } else { + // For other types (like files), just index in a new transaction + indexPermissionsForOneDvObject(definitionPoint); + numObjects++; + } + + logger.info("Reindexed permissions for " + counter[0] + " files and " + numObjects + + " datasets/collections in " + (System.currentTimeMillis() - globalStartTime) + " ms"); + + return new IndexResponse("Number of dvObject permissions indexed for " + definitionPoint + ": " + numObjects); + } + + @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) + public void indexDatasetBatchInNewTransaction(List datasetIds, final int[] fileCounter, int fileQueryMin) { + for (Long datasetId : datasetIds) { + Dataset dataset = datasetService.find(datasetId); + if (dataset != null) { indexPermissionsForOneDvObject(dataset); - numObjects++; + // Process files for this dataset Map desiredCards = searchPermissionsService.getDesiredCards(dataset); - long startTime = System.currentTimeMillis(); + for (DatasetVersion version : versionsToReIndexPermissionsFor(dataset)) { if (desiredCards.get(version.getVersionState())) { - List cachedPerms = searchPermissionsService.findDatasetVersionPerms(version); - String solrIdEnd = getDatasetOrDataFileSolrEnding(version.getVersionState()); - Long versionId = version.getId(); - for (FileMetadata fmd : version.getFileMetadatas()) { - DataFileProxy fileProxy = new DataFileProxy(fmd); - // Since reindexFilesInBatches() re-indexes a file in all versions needed, we should not send a file already in the released version twice - filesToReindexAsBatch.add(fileProxy); - counter[0]++; - if (counter[0] % 100 == 0) { - reindexFilesInBatches(filesToReindexAsBatch, cachedPerms, versionId, solrIdEnd); - filesToReindexAsBatch.clear(); - } - if (counter[0] % 1000 == 0) { - logger.fine("Progress: " + counter[0] + "files permissions reindexed"); - } - } - - // Re-index any remaining files in the datasetversion (so that verionId, etc. remain constants for all files in the batch) - reindexFilesInBatches(filesToReindexAsBatch, cachedPerms, versionId, solrIdEnd); - logger.info("Progress : dataset " + dataset.getId() + " permissions reindexed in " + (System.currentTimeMillis() - startTime) + " ms"); + processDatasetVersionFiles(version, fileCounter, fileQueryMin); } } } - } else if (definitionPoint.isInstanceofDataset()) { - indexPermissionsForOneDvObject(definitionPoint); - numObjects++; - // index files - Dataset dataset = (Dataset) definitionPoint; + } + } + + @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) + public void indexDatasetFilesInNewTransaction(Long datasetId, final int[] fileCounter, int fileQueryMin) { + Dataset dataset = datasetService.find(datasetId); + if (dataset != null) { Map desiredCards = searchPermissionsService.getDesiredCards(dataset); for (DatasetVersion version : versionsToReIndexPermissionsFor(dataset)) { if (desiredCards.get(version.getVersionState())) { - List cachedPerms = searchPermissionsService.findDatasetVersionPerms(version); - String solrIdEnd = getDatasetOrDataFileSolrEnding(version.getVersionState()); - Long versionId = version.getId(); - if (version.getFileMetadatas().size() > fileQueryMin) { - // For large datasets, use a more efficient SQL query instead of loading all file metadata objects - getDataFileInfoForPermissionIndexing(version.getId()).forEach(fileInfo -> { - filesToReindexAsBatch.add(fileInfo); - counter[0]++; - - if (counter[0] % 100 == 0) { - long startTime = System.currentTimeMillis(); - reindexFilesInBatches(filesToReindexAsBatch, cachedPerms, versionId, solrIdEnd); - filesToReindexAsBatch.clear(); - logger.fine("Progress: 100 file permissions at " + counter[0] + " files reindexed in " + (System.currentTimeMillis() - startTime) + " ms"); - } - }); - } else { - version.getFileMetadatas().stream() - .forEach(fmd -> { - DataFileProxy fileProxy = new DataFileProxy(fmd); - filesToReindexAsBatch.add(fileProxy); - counter[0]++; - if (counter[0] % 100 == 0) { - long startTime = System.currentTimeMillis(); - reindexFilesInBatches(filesToReindexAsBatch, cachedPerms, versionId, solrIdEnd); - filesToReindexAsBatch.clear(); - logger.fine("Progress: 100 file permissions at " + counter[0] + "files reindexed in " + (System.currentTimeMillis() - startTime) + " ms"); - } - }); - } - // Re-index any remaining files in the dataset version (versionId, etc. remain constants for all files in the batch) + processDatasetVersionFiles(version, fileCounter, fileQueryMin); + } + } + } + } + + private void processDatasetVersionFiles(DatasetVersion version, + final int[] fileCounter, int fileQueryMin) { + List cachedPerms = searchPermissionsService.findDatasetVersionPerms(version); + String solrIdEnd = getDatasetOrDataFileSolrEnding(version.getVersionState()); + Long versionId = version.getId(); + List filesToReindexAsBatch = new ArrayList<>(); + + // Process files in batches of 100 + int batchSize = 100; + + if (version.getFileMetadatas().size() > fileQueryMin) { + // For large datasets, use a more efficient SQL query + Stream fileStream = getDataFileInfoForPermissionIndexing(version.getId()); + + // Process files in batches to avoid memory issues + fileStream.forEach(fileInfo -> { + filesToReindexAsBatch.add(fileInfo); + fileCounter[0]++; + + if (filesToReindexAsBatch.size() >= batchSize) { reindexFilesInBatches(filesToReindexAsBatch, cachedPerms, versionId, solrIdEnd); filesToReindexAsBatch.clear(); } + }); + } else { + // For smaller datasets, process files directly + for (FileMetadata fmd : version.getFileMetadatas()) { + DataFileProxy fileProxy = new DataFileProxy(fmd); + filesToReindexAsBatch.add(fileProxy); + fileCounter[0]++; + if (filesToReindexAsBatch.size() >= batchSize) { + reindexFilesInBatches(filesToReindexAsBatch, cachedPerms, versionId, solrIdEnd); + filesToReindexAsBatch.clear(); + } } - } else { - indexPermissionsForOneDvObject(definitionPoint); - numObjects++; } - /** - * @todo Error handling? What to do with response? - * - * @todo Should update timestamps, probably, even thought these are files, see - * https://github.com/IQSS/dataverse/issues/2421 - */ - logger.fine("Reindexed permissions for " + counter[0] + " files and " + numObjects + "datasets/collections in " + (System.currentTimeMillis() - globalStartTime) + " ms"); - return new IndexResponse("Number of dvObject permissions indexed for " + definitionPoint - + ": " + numObjects); + // Process any remaining files + if (!filesToReindexAsBatch.isEmpty()) { + reindexFilesInBatches(filesToReindexAsBatch, cachedPerms, versionId, solrIdEnd); + } } private String reindexFilesInBatches(List filesToReindexAsBatch, List cachedPerms, Long versionId, String solrIdEnd) { From 858763ba0c1d9d49522abc13d32e30c689730303 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 16 Sep 2025 14:17:55 -0400 Subject: [PATCH 209/634] return void, update logging messages --- .../search/SolrIndexServiceBean.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index 01b9960e76b..ce3c3ecf7db 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -356,7 +356,7 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) self.indexDatasetBatchInNewTransaction(batchIds, counter, fileQueryMin); numObjects += batchIds.size(); - logger.info("Processed batch " + (i / batchSize + 1) + " of " + + logger.info("Permission reindexing: Processed batch " + (i/batchSize + 1) + " of " + (int) Math.ceil(datasetIds.size() / (double) batchSize) + " dataset batches for dataverse " + dataverse.getId()); } @@ -455,24 +455,25 @@ private void processDatasetVersionFiles(DatasetVersion version, } } - private String reindexFilesInBatches(List filesToReindexAsBatch, List cachedPerms, Long versionId, String solrIdEnd) { + private void reindexFilesInBatches(List filesToReindexAsBatch, List cachedPerms, Long versionId, String solrIdEnd) { List docs = new ArrayList<>(); try { // Assume all files have the same owner if (filesToReindexAsBatch.isEmpty()) { - return "No files to reindex"; + logger.warning("reindexFilesInBatches called incorrectly with an empty file list"); } - for (DataFileProxy file : filesToReindexAsBatch) { + for (DataFileProxy file : filesToReindexAsBatch) { - DvObjectSolrDoc fileSolrDoc = constructDatafileSolrDoc(file, cachedPerms, versionId, solrIdEnd); - SolrInputDocument solrDoc = SearchUtil.createSolrDoc(fileSolrDoc); - docs.add(solrDoc); - } + DvObjectSolrDoc fileSolrDoc = constructDatafileSolrDoc(file, cachedPerms, versionId, solrIdEnd); + SolrInputDocument solrDoc = SearchUtil.createSolrDoc(fileSolrDoc); + docs.add(solrDoc); + } persistToSolr(docs); - return " " + filesToReindexAsBatch.size() + " files indexed across " + docs.size() + " Solr documents "; + logger.fine("Indexed " + filesToReindexAsBatch.size() + " files across " + docs.size() + " Solr documents"); } catch (SolrServerException | IOException ex) { - return " tried to reindex " + filesToReindexAsBatch.size() + " files indexed across " + docs.size() + " Solr documents but caught exception: " + ex; + logger.log(Level.WARNING, "Failed to reindex " + filesToReindexAsBatch.size() + + " files across " + docs.size() + " Solr documents", ex); } } From 113093cce83f049a999a130bfec1e2c7879f873c Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 16 Sep 2025 15:57:44 -0400 Subject: [PATCH 210/634] lower logging to fine --- .../edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index ce3c3ecf7db..aec352a615b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -356,7 +356,7 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) self.indexDatasetBatchInNewTransaction(batchIds, counter, fileQueryMin); numObjects += batchIds.size(); - logger.info("Permission reindexing: Processed batch " + (i/batchSize + 1) + " of " + + logger.fine("Permission reindexing: Processed batch " + (i/batchSize + 1) + " of " + (int) Math.ceil(datasetIds.size() / (double) batchSize) + " dataset batches for dataverse " + dataverse.getId()); } @@ -373,7 +373,7 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) numObjects++; } - logger.info("Reindexed permissions for " + counter[0] + " files and " + numObjects + + logger.fine("Reindexed permissions for " + counter[0] + " files and " + numObjects + " datasets/collections in " + (System.currentTimeMillis() - globalStartTime) + " ms"); return new IndexResponse("Number of dvObject permissions indexed for " + definitionPoint + ": " + numObjects); From c15a8f6c31137685bac88eff601227d6f33caf8c Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 16 Sep 2025 15:57:51 -0400 Subject: [PATCH 211/634] release note --- doc/release-notes/11822-faster-permission-indexing.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/release-notes/11822-faster-permission-indexing.md diff --git a/doc/release-notes/11822-faster-permission-indexing.md b/doc/release-notes/11822-faster-permission-indexing.md new file mode 100644 index 00000000000..f716655232e --- /dev/null +++ b/doc/release-notes/11822-faster-permission-indexing.md @@ -0,0 +1 @@ +Permission reindexing, which occurs, e.g., after a user has been granted a role on a collection, has been made faster and less memory intensive in this release. \ No newline at end of file From 8279e65a2df7f510d0012276acaeef8b38bb2273 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 15 Jul 2025 10:56:57 +0200 Subject: [PATCH 212/634] feat: add `parse` method and unit tests for `SettingsServiceBean.Key` #11639 Implemented a method to parse strings into `SettingsServiceBean.Key` values, handling null and invalid inputs gracefully. Added corresponding unit tests to verify behavior and maintain consistency. --- .../settings/SettingsServiceBean.java | 32 +++++++++++++ .../settings/SettingsServiceBeanTest.java | 48 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index b323a9b7861..60d72785f9b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -694,6 +694,38 @@ Whether Harvesting (OAI) service is enabled public String toString() { return ":" + name(); } + + /** + * Parses the input string to match a corresponding {@code SettingsServiceBean.Key}. + * The method expects the input string to start with a colon (:) followed by the key name. + * If the key name matches one of the existing {@code SettingsServiceBean.Key} enumerations, + * the corresponding key is returned. The check is case-sensitive. + * + * @param key the input string in the format ":KeyName", where "KeyName" corresponds + * to the name of an enumeration in {@code SettingsServiceBean.Key}. + * If {@code key} is null, blank, does not start with a colon (:), or does not + * match any known key, the method returns {@code null}. + * @return the corresponding {@code SettingsServiceBean.Key} if the key matches one + * of the predefined keys, or {@code null} if no match is found. + */ + public static SettingsServiceBean.Key parse(String key) { + // Null safety and format check + if (key == null || key.isBlank() || key.charAt(0) != ':') return null; + + // Cut off the ":" we verified is present before + String normalizedKey = key.substring(1); + + // Iterate through all the known keys and return on match (case sensitive!) + // We are case sensitive here because Dataverse implicitely uses case sensitive keys everywhere! + for (SettingsServiceBean.Key k : SettingsServiceBean.Key.values()) { + if (k.name().equals(normalizedKey)) { + return k; + } + } + + // Fall through on no match + return null; + } } @PersistenceContext diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java new file mode 100644 index 00000000000..8d68f48a5ae --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java @@ -0,0 +1,48 @@ +package edu.harvard.iq.dataverse.settings; + +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class SettingsServiceBeanTest { + + @Nested + class KeyEnumTest { + static List parseTestParameters() { + return List.of( + Arguments.of(null, null), + Arguments.of("", null), + Arguments.of(" ", null), + Arguments.of("foobar", null), + Arguments.of("ShowMuteOptions", null), + Arguments.of(":FooBar", null), + Arguments.of(":ShowMuteOptions", SettingsServiceBean.Key.ShowMuteOptions) + ); + } + + @MethodSource("parseTestParameters") + @ParameterizedTest + void testParse(String sut, SettingsServiceBean.Key expected) { + assertEquals(expected, SettingsServiceBean.Key.parse(sut)); + } + + @Test + void testToString() { + // Make sure we test the intended behavior so it doesn't change by accident. + assertEquals(":ShowMuteOptions", SettingsServiceBean.Key.ShowMuteOptions.toString()); + } + + @Test + void testRoundtrip() { + for (SettingsServiceBean.Key key : SettingsServiceBean.Key.values()) { + assertEquals(key, SettingsServiceBean.Key.parse(key.toString())); + } + } + } +} \ No newline at end of file From 0a9575329a749bea856c1a34d2b5ff173680a9bd Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 15 Jul 2025 11:09:06 +0200 Subject: [PATCH 213/634] feat: add `listAllWithoutLocalizations` method to filter settings without language data #11639 Replaced usage of `listAll` with the new `listAllWithoutLocalizations` method in various parts of the codebase for improved clarity and targeted queries. Added the corresponding named query `Setting.findAllWithoutLang`. Before, listAll was used in context without localization present or dropping the l10n details from outputs. For example, the Admin API to get all settings neglected to share the localized information, making one setting appear multiple times without the l10n information present. This is still subject to change, enabling the API endpoints to keep this information around. --- .../edu/harvard/iq/dataverse/EditDatafilesPage.java | 2 +- .../java/edu/harvard/iq/dataverse/SettingsWrapper.java | 2 +- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 2 +- .../edu/harvard/iq/dataverse/settings/Setting.java | 2 ++ .../iq/dataverse/settings/SettingsServiceBean.java | 10 ++++++++-- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 3fa1c8b2c10..b7d28ceabfa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -383,7 +383,7 @@ public String getHumanPerFormatTabularLimits() { public String populateHumanPerFormatTabularLimits() { String keyPrefix = ":TabularIngestSizeLimit:"; List formatLimits = new ArrayList<>(); - for (Setting setting : settingsService.listAll()) { + for (Setting setting : settingsService.listAllWithoutLocalizations()) { String name = setting.getName(); if (!name.startsWith(keyPrefix)) { continue; diff --git a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java index 3ff27699379..41fc605bfb3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java @@ -218,7 +218,7 @@ public Integer getInteger(String settingKey, Integer defaultValue) { private void initSettingsMap() { // initialize settings map settingsMap = new HashMap<>(); - for (Setting setting : settingsService.listAll()) { + for (Setting setting : settingsService.listAllWithoutLocalizations()) { settingsMap.put(setting.getName(), setting.getContent()); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index d55f582ecae..f6e622304d8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -198,7 +198,7 @@ public class Admin extends AbstractApiBean { @GET public Response listAllSettings() { JsonObjectBuilder bld = jsonObjectBuilder(); - settingsSvc.listAll().forEach(s -> bld.add(s.getName(), s.getContent())); + settingsSvc.listAllWithoutLocalizations().forEach(s -> bld.add(s.getName(), s.getContent())); return ok(bld); } diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java b/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java index b1910a2fbb5..e429d685c3e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java @@ -19,6 +19,8 @@ query="DELETE FROM Setting s WHERE s.name=:name AND s.lang IS NULL"), @NamedQuery( name="Setting.findAll", query="SELECT s FROM Setting s"), + @NamedQuery( name="Setting.findAllWithoutLang", + query="SELECT s FROM Setting s WHERE s.lang IS NULL"), @NamedQuery( name="Setting.findByName", query = "SELECT s FROM Setting s WHERE s.name=:name AND s.lang IS NULL" ), @NamedQuery( name="Setting.deleteByNameAndLang", diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 60d72785f9b..6239c38ad3f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -1009,8 +1009,14 @@ public void delete( String name, String lang ) { .executeUpdate(); } - public Set listAll() { - return new HashSet<>(em.createNamedQuery("Setting.findAll", Setting.class).getResultList()); + /** + * Retrieves all settings that do not have any language localizations. + * This method uses a named query to fetch settings where the language field is null. + * + * @return a set of {@code Setting} objects that do not have language localizations. + */ + public Set listAllWithoutLocalizations() { + return new HashSet<>(em.createNamedQuery("Setting.findAllWithoutLang", Setting.class).getResultList()); } public Map getBaseMetadataLanguageMap(Map languageMap, boolean refresh) { From 54c3f6e9b9858ccfa5be09abc92c53b00e314092 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 15 Jul 2025 14:54:58 +0200 Subject: [PATCH 214/634] feat: implement `listAllAsJson` for settings retrieval with localization support #11639 Replaced `listAllWithoutLocalizations` in the Admin API with the new `listAllAsJson` method, enabling inclusion of localized setting variants in API responses. Added OpenAPI annotations for improved documentation. Introduced comprehensive unit tests to validate behavior of localized and non-localized settings handling. --- .../edu/harvard/iq/dataverse/api/Admin.java | 17 +++- .../settings/SettingsServiceBean.java | 46 +++++++++++ .../settings/SettingsServiceBeanTest.java | 80 +++++++++++++++++++ 3 files changed, 139 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index f6e622304d8..69ed33f60dc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -132,6 +132,11 @@ import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.StreamingOutput; +import org.eclipse.microprofile.openapi.annotations.media.Content; +import org.eclipse.microprofile.openapi.annotations.media.Schema; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponses; + import java.nio.file.Paths; import java.util.TreeMap; @@ -193,13 +198,17 @@ public class Admin extends AbstractApiBean { public static final String listUsersPartialAPIPath = "list-users"; public static final String listUsersFullAPIPath = "/api/admin/" + listUsersPartialAPIPath; - + @Path("settings") @GET + @APIResponses({ + @APIResponse(responseCode = "200", + description = "All database options successfully queried", + // The schema may be extended later to better describe what the JSON object looks like. + content = @Content(schema = @Schema(implementation = JsonObject.class))), + }) public Response listAllSettings() { - JsonObjectBuilder bld = jsonObjectBuilder(); - settingsSvc.listAllWithoutLocalizations().forEach(s -> bld.add(s.getName(), s.getContent())); - return ok(bld); + return ok(settingsSvc.listAllAsJson()); } @Path("settings/{name}") diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 6239c38ad3f..48c37480e0e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -8,8 +8,10 @@ import jakarta.ejb.EJB; import jakarta.ejb.Stateless; import jakarta.inject.Named; +import jakarta.json.Json; import jakarta.json.JsonArray; import jakarta.json.JsonObject; +import jakarta.json.JsonObjectBuilder; import jakarta.json.JsonValue; import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; @@ -28,6 +30,7 @@ import java.util.StringTokenizer; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * Service bean accessing a persistent hash map, used as settings in the application. @@ -1019,6 +1022,49 @@ public Set listAllWithoutLocalizations() { return new HashSet<>(em.createNamedQuery("Setting.findAllWithoutLang", Setting.class).getResultList()); } + /** + * Retrieves all available application settings as a JSON object. + * The method fetches settings from the database, organizes them into localized + * and non-localized entries, and builds a JSON representation of the dataset. + * Non-localized settings are added directly as key-value pairs, while localized + * settings are grouped under their associated keys with language-specific mappings. + * Note: settings may exist with both non-localized and localized variant. + * The non-localized variant will be added as "base" locale. + * + * @return a {@code JsonObject} containing all application settings, organized + * as key-value pairs for non-localized settings, or as sub-objects + * for settings with language localizations. + */ + public JsonObject listAllAsJson() { + Set settings = new HashSet<>(em.createNamedQuery("Setting.findAll", Setting.class).getResultList()); + + Set settingsWithL10n = settings.stream() + .filter(s -> s.getLang() != null) + .map(Setting::getName) + .collect(Collectors.toUnmodifiableSet()); + Map localizedSettings = new HashMap<>(); + + JsonObjectBuilder response = Json.createObjectBuilder(); + + // Iterate over all the settings and add them to the response. + settings.forEach(setting -> { + // Simple case: This settings is not localized, go ahead and add it. + if (!settingsWithL10n.contains(setting.getName())) { + response.add(setting.getName(), setting.getContent()); + // Localized case: We can't just add it, we need to have a sub-object. + // Also, we don't know the order of the settings or when all localized variants are done. + } else { + localizedSettings.computeIfAbsent(setting.getName(), name -> Json.createObjectBuilder()); + localizedSettings.get(setting.getName()) + .add(setting.getLang() == null ? "base" : setting.getLang(), setting.getContent()); + } + }); + + // We now know that we processed all settings, so add all the l10n builders at once. + localizedSettings.forEach(response::add); + return response.build(); + } + public Map getBaseMetadataLanguageMap(Map languageMap, boolean refresh) { if (languageMap == null || refresh) { languageMap = new HashMap(); diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java index 8d68f48a5ae..a80b4786982 100644 --- a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java @@ -1,14 +1,22 @@ package edu.harvard.iq.dataverse.settings; +import jakarta.json.JsonObject; +import jakarta.persistence.EntityManager; +import jakarta.persistence.TypedQuery; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.ArgumentMatchers; +import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; class SettingsServiceBeanTest { @@ -45,4 +53,76 @@ void testRoundtrip() { } } } + + @Nested + class ListAllAsJsonTest { + + static TypedQuery typedQuery = mock(TypedQuery.class); + static EntityManager em = mock(EntityManager.class); + static SettingsServiceBean settingsServiceBean = new SettingsServiceBean(); + + @BeforeAll + static void setup() { + settingsServiceBean.em = em; + + when(em.createNamedQuery( + ArgumentMatchers.eq("Setting.findAll"), + ArgumentMatchers.eq(Setting.class))) + .thenReturn(typedQuery); + } + + @Test + void testListAllAsJson_noSettings() { + // Given + List emptyList = Collections.emptyList(); + when(typedQuery.getResultList()).thenReturn(emptyList); + + // When + JsonObject result = settingsServiceBean.listAllAsJson(); + + // Then + assertEquals(0, result.size()); + } + + @Test + void testListAllAsJson_nonLocalizedSettings() { + // Given + List resultList = List.of( + new Setting("testKey1", "testValue1"), + new Setting("testKey2", "testValue2") + ); + when(typedQuery.getResultList()).thenReturn(resultList); + + // When + JsonObject result = settingsServiceBean.listAllAsJson(); + + // Then + assertEquals(2, result.size()); + assertEquals("testValue1", result.getString("testKey1")); + assertEquals("testValue2", result.getString("testKey2")); + } + + @Test + void testListAllAsJson_localizedSettings() { + // Given + List resultList = List.of( + new Setting("localizedKey", "value_base"), + new Setting("localizedKey", "en", "value_en"), + new Setting("localizedKey", "fr", "value_fr") + ); + when(typedQuery.getResultList()).thenReturn(resultList); + + // When + JsonObject result = settingsServiceBean.listAllAsJson(); + + // Then + assertEquals(1, result.size()); + JsonObject localizedSetting = result.getJsonObject("localizedKey"); + + assertEquals(3, localizedSetting.size()); + assertEquals("value_base", localizedSetting.getString("base")); + assertEquals("value_en", localizedSetting.getString("en")); + assertEquals("value_fr", localizedSetting.getString("fr")); + } + } } \ No newline at end of file From 0ebe683f97435a1350b0c6c1f93db486f8cec6ce Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 15 Jul 2025 17:14:26 +0200 Subject: [PATCH 215/634] chore: add JaCoCo args placeholders to pom.xml This is because Intellij IDEA's MavenJUnitPatcher only correctly uses the maven-surefire-plugin if all interpolated properties are present. If a single property is missing, the will not be used by IntelliJ's test runner, which is horrible to debug. See also https://github.com/kreiger/idea-maven-test-profiles-argline --- pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pom.xml b/pom.xml index ceb5ea28d84..822eaaeb79d 100644 --- a/pom.xml +++ b/pom.xml @@ -21,6 +21,8 @@ false false integration + + From 684c7d1110b7fb7c760b3f54bca3bcb62eafc532 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 15 Jul 2025 17:18:07 +0200 Subject: [PATCH 216/634] feat: replace tabular ingest size limits configuration using JSON #11639 Introduced new methods in `SystemConfig` to support tabular ingest size limits as JSON objects or single values. This replaces the old way of using colon-separated format suffixes with the setting name. Enhanced flexibility with format-specific or universal defaults. Updated `EditDatafilesPage` to populate human-readable format-specific limits. Added comprehensive unit tests to validate behavior, including edge cases for invalid configurations. --- .../iq/dataverse/EditDatafilesPage.java | 20 +-- .../iq/dataverse/util/SystemConfig.java | 131 +++++++++++++----- .../iq/dataverse/EditDatafilesPageTest.java | 64 +++++++++ .../iq/dataverse/util/SystemConfigTest.java | 88 +++++++++++- 4 files changed, 253 insertions(+), 50 deletions(-) create mode 100644 src/test/java/edu/harvard/iq/dataverse/EditDatafilesPageTest.java diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index b7d28ceabfa..fd0f3be9871 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -81,6 +81,8 @@ import java.util.Collection; import java.util.Set; import java.util.logging.Level; +import java.util.stream.Collectors; + import jakarta.faces.event.AjaxBehaviorEvent; import jakarta.faces.event.FacesEvent; import jakarta.servlet.ServletOutputStream; @@ -381,19 +383,11 @@ public String getHumanPerFormatTabularLimits() { } public String populateHumanPerFormatTabularLimits() { - String keyPrefix = ":TabularIngestSizeLimit:"; - List formatLimits = new ArrayList<>(); - for (Setting setting : settingsService.listAllWithoutLocalizations()) { - String name = setting.getName(); - if (!name.startsWith(keyPrefix)) { - continue; - } - String tabularName = setting.getName().substring(keyPrefix.length()); - String bytes = setting.getContent(); - String humanReadableSize = FileSizeChecker.bytesToHumanReadable(Long.valueOf(bytes)); - formatLimits.add(tabularName + ": " + humanReadableSize); - } - return String.join(", ", formatLimits); + return systemConfig.getTabularIngestSizeLimits().entrySet().stream() + // The human-readable list shall not contain the setting for non-matching formats + .filter(entry -> ! entry.getKey().equals(SystemConfig.TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY)) + .map(entry -> entry.getKey() + ": " + FileSizeChecker.bytesToHumanReadable(entry.getValue())) + .collect(Collectors.joining(", ")); } public Integer getFileUploadsAvailable() { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 69f9262ab5b..5061a0a70e7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -11,6 +11,7 @@ import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.validation.PasswordValidatorUtil; +import jakarta.json.stream.JsonParsingException; import org.passay.CharacterRule; import jakarta.ejb.EJB; @@ -27,6 +28,7 @@ import java.net.UnknownHostException; import java.time.Year; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -488,49 +490,106 @@ public Integer getSearchHighlightFragmentSize() { } return null; } - - public long getTabularIngestSizeLimit() { - // This method will return the blanket ingestable size limit, if - // set on the system. I.e., the universal limit that applies to all - // tabular ingests, regardless of fromat: - - String limitEntry = settingsService.getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit); - + + /** + * The default key used to identify tabular ingest size limits. + * This value represents the standard or fallback configuration. + * For any other valid format strings, see implementations of {@code TabularDataFileReader.getFormatName()}. + */ + public static final String TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY = "default"; + + /** + * Retrieves the tabular ingest size limits based on the system configuration. + * The size limits can be defined as a JSON object with format-specific limits, a single numeric value + * applied to all formats, or might not exist, in which case the default limit is applied. + * + * Note that the format names in the configuration will be transformed to lowercase for user convenience + * of how people like to write their formats best. + * + * If the configuration contains invalid data (e.g., unparsable JSON or non-numeric values), + * all tabular ingest operations are disabled by setting size limits to 0. + * + * TODO: At some later point, if and when the DB lookups or JSON parsing takes a toll to heavy to bear, + * we may introduce a caching singleton for these. (With TTL or using events to invalidate on update.) + * + * @return a map where the keys represent format names or a default key, and the values represent the maximum allowed size for each format. + */ + public Map getTabularIngestSizeLimits() { + String limitEntry = settingsService.getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit); if (limitEntry != null) { - try { - Long sizeOption = Long.valueOf(limitEntry); - return sizeOption; - } catch (NumberFormatException nfe) { - logger.warning("Invalid value for TabularIngestSizeLimit option? - " + limitEntry); + // Case A: the setting is using JSON to support multiple formats + if (limitEntry.trim().startsWith("{")) { + try { + JsonObject limits = Json.createReader(new StringReader(limitEntry)).readObject(); + + Map limitsMap = new HashMap<>(); + // We add the default in case the JSON does not contain the default (which is optional). + limitsMap.put(TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY, -1L); + + for (String formatName : limits.keySet()) { + // We deliberatly do not validate the formatNames here for backward compatibility. + // But we transform to lowercase here, so the casing doesn't matter for lookups. + String lowercaseFormatName = formatName.toLowerCase(); + + try { + Long sizeOption = Long.valueOf(limits.getString(formatName)); + limitsMap.put(lowercaseFormatName, sizeOption); + } catch (NumberFormatException nfe) { + logger.warning("Could not convert " + SettingsServiceBean.Key.TabularIngestSizeLimit + " to long: " + nfe); + logger.warning("Disabling all tabular ingest completely until fixed!"); + return Map.of(TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY, 0L); + } + } + + return Collections.unmodifiableMap(limitsMap); + } catch (JsonParsingException e) { + logger.warning("Invalid TabularIngestSizeLimit option found, cannot parse JSON: " + e.getMessage()); + logger.warning("Disabling all tabular ingest completely until fixed!"); + return Map.of(TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY, 0L); + } + // Case B: It might be just a simple Long, providing a default for all formats. + } else { + try { + Long limit = Long.valueOf(limitEntry); + return Map.of(TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY, limit); + } catch (NumberFormatException nfe) { + logger.warning("Could not convert " + SettingsServiceBean.Key.TabularIngestSizeLimit + " to long: " + nfe); + logger.warning("Disabling all tabular ingest completely until fixed!"); + return Map.of(TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY, 0L); + } } } - // -1 means no limit is set; - // 0 on the other hand would mean that ingest is fully disabled for - // tabular data. - return -1; + + // Default is not to limit at all + return Map.of(TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY, -1L); } + /** + * This method will return the blanket ingestable size limit, if set on the system. + * I.e., the universal limit that applies to all tabular ingests, regardless of fromat. + * @return -1 = unlimited if not set, 0 if disabled or invalid, some long number of bytes otherwise + */ + public long getTabularIngestSizeLimit() { + return getTabularIngestSizeLimits().get(TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY); + } + + /** + * Retrieves the size limit for tabular data ingestion based on the provided format name. + * The format name will be converted to lowercase, making sure the casing doesn't matter. + * + * @param formatName The name of the format for which the size limit is requested + * See also implementations of {@code TabularDataFileReader.getFormatName()} for examples. + * @return The size limit in bytes for tabular data ingestion associated with the specified format name, + * or the default size limit if no format-specific limit is found or its name is invalid (null, blank, ...). + * -1 = unlimited if not set, 0 if disabled or invalid, some long number of bytes otherwise + */ public long getTabularIngestSizeLimit(String formatName) { - // This method returns the size limit set specifically for this format name, - // if available, otherwise - the blanket limit that applies to all tabular - // ingests regardless of a format. - - if (formatName == null || formatName.equals("")) { - return getTabularIngestSizeLimit(); + if (formatName != null && !formatName.isBlank()) { + // We convert to lowercase so it doesn't matter which variant someone uses in the JSON config + String convertedFormatName = formatName.toLowerCase(); + return getTabularIngestSizeLimits().getOrDefault(convertedFormatName, getTabularIngestSizeLimit()); } - - String limitEntry = settingsService.get(SettingsServiceBean.Key.TabularIngestSizeLimit.toString() + ":" + formatName); - - if (limitEntry != null) { - try { - Long sizeOption = Long.valueOf(limitEntry); - return sizeOption; - } catch (NumberFormatException nfe) { - logger.warning("Invalid value for TabularIngestSizeLimit:" + formatName + "? - " + limitEntry ); - } - } - - return getTabularIngestSizeLimit(); + return getTabularIngestSizeLimit(); } public boolean isOAIServerEnabled() { diff --git a/src/test/java/edu/harvard/iq/dataverse/EditDatafilesPageTest.java b/src/test/java/edu/harvard/iq/dataverse/EditDatafilesPageTest.java new file mode 100644 index 00000000000..11578b71f0e --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/EditDatafilesPageTest.java @@ -0,0 +1,64 @@ +package edu.harvard.iq.dataverse; + +import edu.harvard.iq.dataverse.util.SystemConfig; +import org.junit.jupiter.api.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +class EditDatafilesPageTest { + + @InjectMocks + private EditDatafilesPage editDatafilesPage; + + @Mock + private SystemConfig systemConfig; + + public EditDatafilesPageTest() { + MockitoAnnotations.openMocks(this); + } + + @Test + void testPopulateHumanPerFormatTabularLimits_WithEmptyLimits() { + Map tabularLimits = new HashMap<>(); + when(systemConfig.getTabularIngestSizeLimits()).thenReturn(tabularLimits); + + String result = editDatafilesPage.populateHumanPerFormatTabularLimits(); + + assertEquals("", result, "Expected no formatted limits when the map is empty"); + } + + @Test + void testPopulateHumanPerFormatTabularLimits_WithNonDefaultLimits() { + Map tabularLimits = new HashMap<>(); + tabularLimits.put("csv", 10485760L); // 10MB + tabularLimits.put("tsv", 5242880L); // 5MB + when(systemConfig.getTabularIngestSizeLimits()).thenReturn(tabularLimits); + + String result = editDatafilesPage.populateHumanPerFormatTabularLimits(); + + assertTrue(result.contains("csv: 10.0 MB"), "Expected CSV limit in human-readable format, but got: " + result); + assertTrue(result.contains("tsv: 5.0 MB"), "Expected TSV limit in human-readable format, but got: " + result); + } + + @Test + void testPopulateHumanPerFormatTabularLimits_WithDefaultKey() { + Map tabularLimits = new HashMap<>(); + tabularLimits.put(SystemConfig.TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY, 2097152L); // 2MB + tabularLimits.put("csv", 10485760L); // 10MB + when(systemConfig.getTabularIngestSizeLimits()).thenReturn(tabularLimits); + + String result = editDatafilesPage.populateHumanPerFormatTabularLimits(); + + assertTrue(result.contains("csv: 10.0 MB"), "Expected CSV limit in human-readable format, but got: " + result); + assertFalse(result.contains("default"), "Default key should be excluded from the output"); + } +} \ No newline at end of file diff --git a/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java b/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java index 82b89bca678..f50b6023480 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java @@ -12,6 +12,8 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.doReturn; @@ -142,5 +144,89 @@ void testGetThumbnailSizeLimit() { assertEquals(1000000l, SystemConfig.getThumbnailSizeLimit("PDF")); assertEquals(0l, SystemConfig.getThumbnailSizeLimit("NoSuchType")); } - + + @Test + void testGetTabularIngestSizeLimitsWithoutSetting() { + // given + doReturn(null).when(settingsService).getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit); + + // when + Map result = systemConfig.getTabularIngestSizeLimits(); + + // then + assertEquals(1, result.size()); + assertEquals(-1L, (long) result.get(SystemConfig.TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY)); + } + + @Test + void testGetTabularIngestSizeLimitsWithValidJson() { + // given + String validJson = "{\"csV\": \"5000\", \"tSv\": \"10000\"}"; + doReturn(validJson).when(settingsService).getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit); + + // when + Map result = systemConfig.getTabularIngestSizeLimits(); + + // then + assertEquals(3, result.size()); + assertEquals(-1L, (long) result.get(SystemConfig.TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY)); + assertEquals(5000L, result.get("csv")); + assertEquals(10000L, result.get("tsv")); + } + + @Test + void testGetTabularIngestSizeLimitsWithSingleValue() { + // given + String singleValue = "8000"; + doReturn(singleValue).when(settingsService).getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit); + + // when + Map result = systemConfig.getTabularIngestSizeLimits(); + + // then + assertEquals(1, result.size()); + assertEquals(8000L, (long) result.get(SystemConfig.TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY)); + } + + @Test + void testGetTabularIngestSizeLimitsWithSingleInvalidValue() { + // given + String singleValue = "this-aint-no-number"; + doReturn(singleValue).when(settingsService).getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit); + + // when + Map result = systemConfig.getTabularIngestSizeLimits(); + + // then + assertEquals(1, result.size()); + assertEquals(0L, (long) result.get(SystemConfig.TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY)); + } + + @Test + void testGetTabularIngestSizeLimitsWithInvalidJson() { + // given + String invalidJson = "{invalid:}"; + doReturn(invalidJson).when(settingsService).getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit); + + // when + Map result = systemConfig.getTabularIngestSizeLimits(); + + // then + assertEquals(1, result.size()); + assertEquals(0L, (long) result.get(SystemConfig.TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY)); + } + + @Test + void testGetTabularIngestSizeLimitsWithInvalidNumberInValidJson() { + // given + String invalidJson = "{\"csv\": \"this-is-not-a-number\", \"tSv\": \"10000\"}"; + doReturn(invalidJson).when(settingsService).getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit); + + // when + Map result = systemConfig.getTabularIngestSizeLimits(); + + // then + assertEquals(1, result.size()); + assertEquals(0L, (long) result.get(SystemConfig.TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY)); + } } From b11b722ed75f0feb19858a3d2bb264ecf269d62d Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 15 Jul 2025 17:52:55 +0200 Subject: [PATCH 217/634] feat(db): migrate TabularIngestSizeLimit settings to JSON format with Flyway #11639 Implemented migration script to replace old format-specific TabularIngestSizeLimit settings with a unified JSON-based structure. Validates and handles non-numeric values, ensures backward compatibility, and cleans up obsolete settings. --- src/main/resources/db/migration/V6.7.0.1.sql | 78 ++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 src/main/resources/db/migration/V6.7.0.1.sql diff --git a/src/main/resources/db/migration/V6.7.0.1.sql b/src/main/resources/db/migration/V6.7.0.1.sql new file mode 100644 index 00000000000..733a10d951f --- /dev/null +++ b/src/main/resources/db/migration/V6.7.0.1.sql @@ -0,0 +1,78 @@ +-- This script migrates the old TabularIngestSizeLimit database setting using format suffixes to a JSON based approach. + +DO $$ + DECLARE + base_setting_content TEXT; + format_settings_cursor CURSOR FOR + SELECT name, content + FROM Setting + WHERE name LIKE ':TabularIngestSizeLimit:%' + AND lang IS NULL; + format_record RECORD; + format_name TEXT; + format_value BIGINT; + json_object JSONB := '{}'; + has_format_settings BOOLEAN := FALSE; + warning_message TEXT; + BEGIN + -- Check if there are any format-specific settings + SELECT EXISTS( + SELECT 1 FROM Setting + WHERE name LIKE ':TabularIngestSizeLimit:%' + AND lang IS NULL + ) INTO has_format_settings; + + -- Only proceed if we have format-specific settings + IF NOT has_format_settings THEN + RAISE NOTICE 'No format-specific TabularIngestSizeLimit settings found. Skipping migration.'; + RETURN; + END IF; + + -- Get the base setting (without format suffix) if it exists + SELECT content INTO base_setting_content + FROM Setting + WHERE name = ':TabularIngestSizeLimit' + AND lang IS NULL; + + -- Add base setting to JSON object if it exists + IF base_setting_content IS NOT NULL THEN + -- Validate that base setting is numeric + BEGIN + format_value := base_setting_content::BIGINT; + json_object := json_object || jsonb_build_object('default', format_value); + EXCEPTION WHEN invalid_text_representation THEN + RAISE WARNING 'Base TabularIngestSizeLimit setting contains non-numeric value: %. Setting to 0 (disabling ingest!).', base_setting_content; + json_object := json_object || jsonb_build_object('default', 0); + END; + END IF; + + -- Process format-specific settings + FOR format_record IN format_settings_cursor LOOP + -- Extract format name (everything after ":TabularIngestSizeLimit:") + format_name := substring(format_record.name from ':TabularIngestSizeLimit:(.*)'); + + -- Validate and convert the content to numeric + BEGIN + format_value := format_record.content::BIGINT; + json_object := json_object || jsonb_build_object(format_name, format_value); + EXCEPTION WHEN invalid_text_representation THEN + warning_message := format('Format-specific TabularIngestSizeLimit setting %s contains non-numeric value: %s. Setting to 0 (disabling ingest!).', + format_record.name, format_record.content); + RAISE WARNING '%', warning_message; + json_object := json_object || jsonb_build_object(format_name, 0); + END; + END LOOP; + + -- Insert or update the new JSON-based setting + INSERT INTO Setting (name, content, lang) + VALUES (':TabularIngestSizeLimit', json_object::TEXT, NULL) + ON CONFLICT (name) WHERE lang IS NULL + DO UPDATE SET content = EXCLUDED.content; + + -- Delete all format-specific settings + DELETE FROM Setting + WHERE name LIKE ':TabularIngestSizeLimit:%' + AND lang IS NULL; + + RAISE NOTICE 'Successfully migrated TabularIngestSizeLimit settings to JSON format: %', json_object::TEXT; + END $$; \ No newline at end of file From 92adb8ada567a73f67605e4618a8642c18db6329 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 16 Jul 2025 11:00:58 +0200 Subject: [PATCH 218/634] docs: improve TabularIngestSizeLimit documentation with JSON examples and defaults #11639 Refined the explanation of tabular ingest size limits, introduced examples for JSON configuration, clarified default behavior, and updated guidance on per-format overrides. --- .../source/installation/config.rst | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 14bf33c9482..63eb4612f7f 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -4373,30 +4373,45 @@ In the UI, users trying to download a zip file larger than the Dataverse install :TabularIngestSizeLimit +++++++++++++++++++++++ -Threshold in bytes for limiting whether or not "ingest" it attempted for tabular files (which can be resource intensive). For example, with the below in place, files greater than 2 GB in size will not go through the ingest process: +Threshold in bytes for limiting whether or not "ingest" is attempted for an uploaded tabular file (which can be resource intensive). +For example, with the below in place, files greater than 2 GB in size will not go through the ingest process: ``curl -X PUT -d 2000000000 http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit`` -(You can set this value to 0 to prevent files from being ingested at all.) +You can set this value to ``0`` to prevent files from being ingested at all. +The default is ``-1``, meaning no file size limit is applied. -You can override this global setting on a per-format basis for the following formats: +Using a JSON-based setting, you can override this global setting on a per-format basis for the following formats: - DTA - POR - SAV - Rdata - CSV -- XLSX (in lower-case) +- XLSX -For example : +The JSON follows this form, all fields optional: -* if you want your Dataverse installation to not attempt to ingest Rdata files larger than 1 MB, use this setting: +.. code:: json -``curl -X PUT -d 1000000 http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit:Rdata`` + { + "default": -1, + "formatX": 0, + "formatY": 10, + "formatZ": 100 + } + +The ``default`` key represents the global default, with it being absent meaning the global default of ``-1`` applies. +Add a format name (as listed above) to change the limit for this particular format. + +Examples: + +1. If you want your Dataverse installation to not attempt to ingest Rdata files larger than 1 MB but otherwise unlimited: -* if you want your Dataverse installation to not attempt to ingest XLSX files at all, use this setting: + ``curl -X PUT -d '{"Rdata":1000000}' http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit`` +2. If you want your Dataverse installation to not attempt to ingest XLSX files at all and apply a global limit of 512 MiB, use this setting: -``curl -X PUT -d 0 http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit:xlsx`` + ``curl -X PUT -d '{"default":536870912, "XSLX":0}' http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit`` :ZipUploadFilesLimit ++++++++++++++++++++ From b17e52a444f09186cd20ba57c4c22204fb0343c1 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 16 Jul 2025 12:06:39 +0200 Subject: [PATCH 219/634] fix: handle unsupported JSON integers in tabular ingest size limits #11639 Updated warnings and validation logic to enforce string literals for size limits. Added a new test to ensure proper handling of JSON configurations with unsupported integer types for tabular ingest size limits. Improved related documentation for clarity. --- doc/sphinx-guides/source/installation/config.rst | 15 ++++++++------- .../harvard/iq/dataverse/util/SystemConfig.java | 6 +++++- .../iq/dataverse/util/SystemConfigTest.java | 14 ++++++++++++++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 63eb4612f7f..0541f9f301d 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -4395,23 +4395,24 @@ The JSON follows this form, all fields optional: .. code:: json { - "default": -1, - "formatX": 0, - "formatY": 10, - "formatZ": 100 + "default": "-1", + "formatX": "0", + "formatY": "10", + "formatZ": "100" } -The ``default`` key represents the global default, with it being absent meaning the global default of ``-1`` applies. +The ``default`` key represents the global default (with it being absent meaning the implicit global default of ``-1`` applies). Add a format name (as listed above) to change the limit for this particular format. +Any size limits must be provided as string literals (in quotes), not number literals! Examples: 1. If you want your Dataverse installation to not attempt to ingest Rdata files larger than 1 MB but otherwise unlimited: - ``curl -X PUT -d '{"Rdata":1000000}' http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit`` + ``curl -X PUT -d '{"Rdata":"1000000"}' http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit`` 2. If you want your Dataverse installation to not attempt to ingest XLSX files at all and apply a global limit of 512 MiB, use this setting: - ``curl -X PUT -d '{"default":536870912, "XSLX":0}' http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit`` + ``curl -X PUT -d '{"default":"536870912", "XSLX":"0"}' http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit`` :ZipUploadFilesLimit ++++++++++++++++++++ diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 5061a0a70e7..71f24b1fe3a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -534,8 +534,12 @@ public Map getTabularIngestSizeLimits() { try { Long sizeOption = Long.valueOf(limits.getString(formatName)); limitsMap.put(lowercaseFormatName, sizeOption); + } catch (ClassCastException cce) { + logger.warning("Could not convert " + SettingsServiceBean.Key.TabularIngestSizeLimit + " to long from JSON integer. You must provide the long number as string (use quotes) for format " + formatName); + logger.warning("Disabling all tabular ingest completely until fixed!"); + return Map.of(TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY, 0L); } catch (NumberFormatException nfe) { - logger.warning("Could not convert " + SettingsServiceBean.Key.TabularIngestSizeLimit + " to long: " + nfe); + logger.warning("Could not convert " + SettingsServiceBean.Key.TabularIngestSizeLimit + " to long for format " + formatName + " (not a number)"); logger.warning("Disabling all tabular ingest completely until fixed!"); return Map.of(TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY, 0L); } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java b/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java index f50b6023480..06026962d2c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java @@ -202,6 +202,20 @@ void testGetTabularIngestSizeLimitsWithSingleInvalidValue() { assertEquals(0L, (long) result.get(SystemConfig.TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY)); } + @Test + void testGetTabularIngestSizeLimitsWithJsonButUnsupportedJsonInt() { + // given + String invalidJson = "{\"default\": 0}"; + doReturn(invalidJson).when(settingsService).getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit); + + // when + Map result = systemConfig.getTabularIngestSizeLimits(); + + // then + assertEquals(1, result.size()); + assertEquals(0L, (long) result.get(SystemConfig.TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY)); + } + @Test void testGetTabularIngestSizeLimitsWithInvalidJson() { // given From 493239652eae37a84ed2237e89234be792601d99 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 16 Jul 2025 17:14:41 +0200 Subject: [PATCH 220/634] refactor: simplify `listAllAsJson` method and update corresponding test #11639 Streamlined the logic of `listAllAsJson` to use a flattened structure for localized settings, improving maintainability and backward compatibility. Updated test assertions in `SettingsServiceBeanTest` to reflect the new structure. Added enhanced method documentation. --- .../settings/SettingsServiceBean.java | 64 ++++++++++--------- .../settings/SettingsServiceBeanTest.java | 11 ++-- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 48c37480e0e..0f3d9557e6b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -1016,52 +1016,56 @@ public void delete( String name, String lang ) { * Retrieves all settings that do not have any language localizations. * This method uses a named query to fetch settings where the language field is null. * - * @return a set of {@code Setting} objects that do not have language localizations. + * @return a set of {@link Setting} objects that do not have language localizations. */ public Set listAllWithoutLocalizations() { return new HashSet<>(em.createNamedQuery("Setting.findAllWithoutLang", Setting.class).getResultList()); } /** - * Retrieves all available application settings as a JSON object. - * The method fetches settings from the database, organizes them into localized - * and non-localized entries, and builds a JSON representation of the dataset. - * Non-localized settings are added directly as key-value pairs, while localized - * settings are grouped under their associated keys with language-specific mappings. - * Note: settings may exist with both non-localized and localized variant. - * The non-localized variant will be added as "base" locale. + * Retrieves all settings from the database and converts them into a JSON object. + * Each setting is represented as a key-value pair in the JSON object. The key + * is the setting name, optionally appended with the language if the setting is + * language-specific, while the value corresponds to the setting's content. * - * @return a {@code JsonObject} containing all application settings, organized - * as key-value pairs for non-localized settings, or as sub-objects - * for settings with language localizations. + * @return A {@link JsonObject} containing all settings from the database, structured + * with their names (and languages, if applicable) as keys and their + * respective contents as values. + * Shortened Example: + * + * { + * ":FilePIDsEnabled": "false", + * ":ApplicationTermsOfUse": "Non-localized default / fallback terms.", + * ":ApplicationTermsOfUse/lang/fr": "Il s'agit de termes localisés en français.", + * ":MaxFileUploadSizeInBytes": { + * "default": "2147483648", + * "fileOne": "4000000000", + * "s3": "8000000000" + * } + * } + * + * + * @implNote The reason to use a flattened approach for the localized settings is to stay backward compatible. + * Per good practice, a bulk operation should be a composite of the single operation. + * As you need to provide the language parameter to query or put them single, the localization is not + * part of the content model, but of the {@link Setting} data model. Using a JSON sub-object or using + * a separated approach is possible, but adds additional complexity. In case of the sub-object it even + * violates that the value you retrieve from the bulk operation can be used for a single operation again. + * As long as we do not update our content model, but store the language as part of the data model, + * this flattening seems to be the most balanced compromise. */ public JsonObject listAllAsJson() { Set settings = new HashSet<>(em.createNamedQuery("Setting.findAll", Setting.class).getResultList()); - - Set settingsWithL10n = settings.stream() - .filter(s -> s.getLang() != null) - .map(Setting::getName) - .collect(Collectors.toUnmodifiableSet()); - Map localizedSettings = new HashMap<>(); - JsonObjectBuilder response = Json.createObjectBuilder(); // Iterate over all the settings and add them to the response. settings.forEach(setting -> { - // Simple case: This settings is not localized, go ahead and add it. - if (!settingsWithL10n.contains(setting.getName())) { - response.add(setting.getName(), setting.getContent()); - // Localized case: We can't just add it, we need to have a sub-object. - // Also, we don't know the order of the settings or when all localized variants are done. - } else { - localizedSettings.computeIfAbsent(setting.getName(), name -> Json.createObjectBuilder()); - localizedSettings.get(setting.getName()) - .add(setting.getLang() == null ? "base" : setting.getLang(), setting.getContent()); - } + response.add( + setting.getName() + (setting.getLang() == null ? "" : "/lang/"+setting.getLang()), + setting.getContent() + ); }); - // We now know that we processed all settings, so add all the l10n builders at once. - localizedSettings.forEach(response::add); return response.build(); } diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java index a80b4786982..47feea5a2fa 100644 --- a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java @@ -116,13 +116,10 @@ void testListAllAsJson_localizedSettings() { JsonObject result = settingsServiceBean.listAllAsJson(); // Then - assertEquals(1, result.size()); - JsonObject localizedSetting = result.getJsonObject("localizedKey"); - - assertEquals(3, localizedSetting.size()); - assertEquals("value_base", localizedSetting.getString("base")); - assertEquals("value_en", localizedSetting.getString("en")); - assertEquals("value_fr", localizedSetting.getString("fr")); + assertEquals(3, result.size()); + assertEquals("value_base", result.getString("localizedKey")); + assertEquals("value_en", result.getString("localizedKey/lang/en")); + assertEquals("value_fr", result.getString("localizedKey/lang/fr")); } } } \ No newline at end of file From d153cacbd3dd311fecc55f90fabc206b002982d0 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 16 Jul 2025 17:26:31 +0200 Subject: [PATCH 221/634] feat: add validation for settings names and language codes in Admin Settings API #11639 Introduced validation logic for setting names and ISO 639-1 language codes across Admin API endpoints. Ensures meaningful error messages for invalid inputs, improving robustness and user feedback. Refactored related methods accordingly. Added missing GET endpoint for localized settings. --- .../edu/harvard/iq/dataverse/api/Admin.java | 86 ++++++++++++++++--- 1 file changed, 74 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 69ed33f60dc..6add116834d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -70,6 +70,7 @@ import java.io.StringReader; import java.nio.charset.StandardCharsets; import java.util.Collections; +import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.function.Predicate; @@ -210,42 +211,103 @@ public class Admin extends AbstractApiBean { public Response listAllSettings() { return ok(settingsSvc.listAllAsJson()); } + + private void validateSettingName(String name) throws IllegalArgumentException { + if (SettingsServiceBean.Key.parse(name) == null) { + // If there is more than one colon, this may be someone trying to use the old suffix settings. + // Change the error message for that slightly. + if (name.replace(":","").length() < name.length() - 1) { + throw new IllegalArgumentException("The name of the setting may not have a colon separated suffix since Dataverse 6.8. Please update your scripts."); + } + throw new IllegalArgumentException("The name of the setting is required."); + } + } + + private void validateSettingLang(String lang) throws IllegalArgumentException { + if (lang == null || lang.length() != 2 || !Arrays.asList(Locale.getISOLanguages()).contains(lang)) { + throw new IllegalArgumentException("The language '" + lang + "' is not a valid ISO 639-1 language code."); + } + } @Path("settings/{name}") @PUT public Response putSetting(@PathParam("name") String name, String content) { - Setting s = settingsSvc.set(name, content); - return ok(jsonObjectBuilder().add(s.getName(), s.getContent())); + try { + validateSettingName(name); + + Setting s = settingsSvc.set(name, content); + return ok("Setting " + name + " added."); + } catch (IllegalArgumentException iae) { + return error(Response.Status.BAD_REQUEST, iae.getMessage()); + } } @Path("settings/{name}/lang/{lang}") @PUT public Response putSettingLang(@PathParam("name") String name, @PathParam("lang") String lang, String content) { - Setting s = settingsSvc.set(name, lang, content); - return ok("Setting " + name + " - " + lang + " - added."); + try { + validateSettingName(name); + validateSettingLang(lang); + + Setting s = settingsSvc.set(name, lang, content); + return ok("Setting " + name + " added for language " + lang + "."); + } catch (IllegalArgumentException iae) { + return error(Response.Status.BAD_REQUEST, iae.getMessage()); + } } @Path("settings/{name}") @GET public Response getSetting(@PathParam("name") String name) { - String s = settingsSvc.get(name); - - return (s != null) ? ok(s) : notFound("Setting " + name + " not found"); + try { + validateSettingName(name); + + String content = settingsSvc.get(name); + return (content != null) ? ok(content) : notFound("Setting " + name + " not found."); + } catch (IllegalArgumentException iae) { + return error(Response.Status.BAD_REQUEST, iae.getMessage()); + } + } + + @Path("settings/{name}/lang/{lang}") + @GET + public Response getSetting(@PathParam("name") String name, @PathParam("lang") String lang) { + try { + validateSettingName(name); + validateSettingLang(lang); + + String content = settingsSvc.get(name, lang); + return (content != null) ? ok(content) : notFound("Setting " + name + " for language " + lang + " not found."); + } catch (IllegalArgumentException iae) { + return error(Response.Status.BAD_REQUEST, iae.getMessage()); + } } @Path("settings/{name}") @DELETE public Response deleteSetting(@PathParam("name") String name) { - settingsSvc.delete(name); - - return ok("Setting " + name + " deleted."); + try { + validateSettingName(name); + + settingsSvc.delete(name); + return ok("Setting " + name + " deleted."); + } catch (IllegalArgumentException iae) { + return error(Response.Status.BAD_REQUEST, iae.getMessage()); + } } @Path("settings/{name}/lang/{lang}") @DELETE public Response deleteSettingLang(@PathParam("name") String name, @PathParam("lang") String lang) { - settingsSvc.delete(name, lang); - return ok("Setting " + name + " - " + lang + " deleted."); + try { + validateSettingName(name); + validateSettingLang(lang); + + settingsSvc.delete(name, lang); + return ok("Setting " + name + " for language " + lang + " deleted."); + } catch (IllegalArgumentException iae) { + return error(Response.Status.BAD_REQUEST, iae.getMessage()); + } } @Path("template/{id}") From 3975db076923d8ae9e82337575e61b6e16512ad8 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 17 Jul 2025 17:07:54 +0200 Subject: [PATCH 222/634] feat: support JSON objects in `listAllAsJson` settings output and update tests #11639 Enhanced `listAllAsJson` to handle settings with JSON content as proper JSON objects in the API response. Introduced a constant for localization key separator. Updated unit tests to cover new behavior with JSON settings. --- .../settings/SettingsServiceBean.java | 16 ++++++++---- .../settings/SettingsServiceBeanTest.java | 26 +++++++++++++++++-- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 0f3d9557e6b..365362d577c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -1022,6 +1022,8 @@ public Set listAllWithoutLocalizations() { return new HashSet<>(em.createNamedQuery("Setting.findAllWithoutLang", Setting.class).getResultList()); } + public static final String L10N_KEY_SEPARATOR = "/lang/"; + /** * Retrieves all settings from the database and converts them into a JSON object. * Each setting is represented as a key-value pair in the JSON object. The key @@ -1060,11 +1062,15 @@ public JsonObject listAllAsJson() { // Iterate over all the settings and add them to the response. settings.forEach(setting -> { - response.add( - setting.getName() + (setting.getLang() == null ? "" : "/lang/"+setting.getLang()), - setting.getContent() - ); - }); + String name = setting.getName() + (setting.getLang() == null ? "" : L10N_KEY_SEPARATOR + setting.getLang()); + + // In case the setting is a JSON object, treat it a such in the output (so the API can return valid JSON) + if (setting.getContent().trim().startsWith("{")) + response.add(name, Json.createObjectBuilder(JsonUtil.getJsonObject(setting.getContent()))); + else + response.add(name, setting.getContent()); + } + ); return response.build(); } diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java index 47feea5a2fa..e52c7ea7887 100644 --- a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java @@ -89,7 +89,7 @@ void testListAllAsJson_nonLocalizedSettings() { // Given List resultList = List.of( new Setting("testKey1", "testValue1"), - new Setting("testKey2", "testValue2") + new Setting("testKey2", "12345") ); when(typedQuery.getResultList()).thenReturn(resultList); @@ -99,7 +99,29 @@ void testListAllAsJson_nonLocalizedSettings() { // Then assertEquals(2, result.size()); assertEquals("testValue1", result.getString("testKey1")); - assertEquals("testValue2", result.getString("testKey2")); + assertEquals("12345", result.getString("testKey2")); + } + + @Test + void testListAllAsJson_jsonSetting() { + // Given + JsonObject expected = Json.createObjectBuilder() + .add("default", "2147483648") + .add("fileOne", "4000000000") + .add("s3", "8000000000") + .build(); + + List resultList = List.of( + new Setting(SettingsServiceBean.Key.MaxFileUploadSizeInBytes.toString(), "{\"default\":\"2147483648\",\"fileOne\":\"4000000000\",\"s3\":\"8000000000\"}") + ); + when(typedQuery.getResultList()).thenReturn(resultList); + + // When + JsonObject result = settingsServiceBean.listAllAsJson(); + + // Then + assertEquals(1, result.size()); + assertEquals(expected.toString(), result.getJsonObject(SettingsServiceBean.Key.MaxFileUploadSizeInBytes.toString()).toString()); } @Test From 865ed5cc4dab2a9c283cfdce8b521896f3a5c0be Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 17 Jul 2025 17:13:03 +0200 Subject: [PATCH 223/634] refactor: centralize validation logic for settings names and language codes in `SettingsServiceBean` #11639 Moved `validateSettingName` and `validateSettingLang` from `Admin` to `SettingsServiceBean` to improve reusability and maintain consistency. Updated tests and API endpoints to use the centralized methods. Expanded test coverage for validation scenarios. --- .../edu/harvard/iq/dataverse/api/Admin.java | 36 ++++--------- .../settings/SettingsServiceBean.java | 40 +++++++++++++++ .../settings/SettingsServiceBeanTest.java | 51 +++++++++++++++++++ 3 files changed, 100 insertions(+), 27 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 6add116834d..fbab4e529b1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -70,7 +70,6 @@ import java.io.StringReader; import java.nio.charset.StandardCharsets; import java.util.Collections; -import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.function.Predicate; @@ -212,28 +211,11 @@ public Response listAllSettings() { return ok(settingsSvc.listAllAsJson()); } - private void validateSettingName(String name) throws IllegalArgumentException { - if (SettingsServiceBean.Key.parse(name) == null) { - // If there is more than one colon, this may be someone trying to use the old suffix settings. - // Change the error message for that slightly. - if (name.replace(":","").length() < name.length() - 1) { - throw new IllegalArgumentException("The name of the setting may not have a colon separated suffix since Dataverse 6.8. Please update your scripts."); - } - throw new IllegalArgumentException("The name of the setting is required."); - } - } - - private void validateSettingLang(String lang) throws IllegalArgumentException { - if (lang == null || lang.length() != 2 || !Arrays.asList(Locale.getISOLanguages()).contains(lang)) { - throw new IllegalArgumentException("The language '" + lang + "' is not a valid ISO 639-1 language code."); - } - } - @Path("settings/{name}") @PUT public Response putSetting(@PathParam("name") String name, String content) { try { - validateSettingName(name); + SettingsServiceBean.validateSettingName(name); Setting s = settingsSvc.set(name, content); return ok("Setting " + name + " added."); @@ -246,8 +228,8 @@ public Response putSetting(@PathParam("name") String name, String content) { @PUT public Response putSettingLang(@PathParam("name") String name, @PathParam("lang") String lang, String content) { try { - validateSettingName(name); - validateSettingLang(lang); + SettingsServiceBean.validateSettingName(name); + SettingsServiceBean.validateSettingLang(lang); Setting s = settingsSvc.set(name, lang, content); return ok("Setting " + name + " added for language " + lang + "."); @@ -260,7 +242,7 @@ public Response putSettingLang(@PathParam("name") String name, @PathParam("lang" @GET public Response getSetting(@PathParam("name") String name) { try { - validateSettingName(name); + SettingsServiceBean.validateSettingName(name); String content = settingsSvc.get(name); return (content != null) ? ok(content) : notFound("Setting " + name + " not found."); @@ -273,8 +255,8 @@ public Response getSetting(@PathParam("name") String name) { @GET public Response getSetting(@PathParam("name") String name, @PathParam("lang") String lang) { try { - validateSettingName(name); - validateSettingLang(lang); + SettingsServiceBean.validateSettingName(name); + SettingsServiceBean.validateSettingLang(lang); String content = settingsSvc.get(name, lang); return (content != null) ? ok(content) : notFound("Setting " + name + " for language " + lang + " not found."); @@ -287,7 +269,7 @@ public Response getSetting(@PathParam("name") String name, @PathParam("lang") St @DELETE public Response deleteSetting(@PathParam("name") String name) { try { - validateSettingName(name); + SettingsServiceBean.validateSettingName(name); settingsSvc.delete(name); return ok("Setting " + name + " deleted."); @@ -300,8 +282,8 @@ public Response deleteSetting(@PathParam("name") String name) { @DELETE public Response deleteSettingLang(@PathParam("name") String name, @PathParam("lang") String lang) { try { - validateSettingName(name); - validateSettingLang(lang); + SettingsServiceBean.validateSettingName(name); + SettingsServiceBean.validateSettingLang(lang); settingsSvc.delete(name, lang); return ok("Setting " + name + " for language " + lang + " deleted."); diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 365362d577c..732b2ffad49 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -20,11 +20,13 @@ import org.json.JSONException; import org.json.JSONObject; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.StringTokenizer; @@ -1123,4 +1125,42 @@ public Set getConfiguredLanguages() { return langs; } + + /** + * Validates the provided setting name to ensure it meets the required format. + * Throws an {@code IllegalArgumentException} if the name is invalid, including cases + * where it contains a colon-separated suffix that is no longer supported. + * + * @param name The name of the setting to be validated. + * It must adhere to the allowable setting name format. + * Names with more than one colon, which may indicate deprecated suffix formats, are not allowed. + * @throws IllegalArgumentException if the setting name is invalid. + */ + public static void validateSettingName(String name) { + if (SettingsServiceBean.Key.parse(name) == null) { + // If there is more than one colon, this may be someone trying to use the old suffix settings. + // Change the error message for that slightly. + if (name.replace(":","").length() < name.length() - 1) { + throw new IllegalArgumentException("The name of the setting may not have a colon separated suffix since Dataverse 6.8. Please update your scripts."); + } + throw new IllegalArgumentException("The name of the setting is invalid."); + } + } + + /** + * Validates the provided language code to ensure it adheres to the ISO 639-1 format. + * This method checks that the language code is not null, has a length of 2 characters, + * and exists within the list of valid ISO 639-1 language codes. If the validation + * fails, an {@code IllegalArgumentException} is thrown. + * + * @param lang the language code to be validated. It must be a non-null, + * 2-character string representing a valid ISO 639-1 language code. + * @throws IllegalArgumentException if the language code is invalid. + */ + public static void validateSettingLang(String lang) { + if (lang == null || lang.length() != 2 || !Arrays.asList(Locale.getISOLanguages()).contains(lang)) { + throw new IllegalArgumentException("The language '" + lang + "' is not a valid ISO 639-1 language code."); + } + } + } diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java index e52c7ea7887..3f65e5317a4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.settings; +import jakarta.json.Json; import jakarta.json.JsonObject; import jakarta.persistence.EntityManager; import jakarta.persistence.TypedQuery; @@ -8,13 +9,17 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentMatchers; import java.util.Collections; import java.util.List; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -54,6 +59,52 @@ void testRoundtrip() { } } + @Nested + class ValidateSettingNameTest { + + @ValueSource(strings = {":ShowMuteOptions", ":AllowApiTokenLookupViaApi", ":OAuth2CallbackUrl"}) + @ParameterizedTest + void testValidateSettingName_validNames(String name) { + assertDoesNotThrow(() -> SettingsServiceBean.validateSettingName(name)); + } + + @CsvSource({ + "invalidName, 'The name of the setting is invalid.'", + ":invalid:suffix, 'The name of the setting may not have a colon separated suffix since Dataverse 6.8. Please update your scripts.'", + ":NonExistentKey, 'The name of the setting is invalid.'", + ":ShowMuteOptions/lang/en, 'The name of the setting is invalid.'" + }) + @ParameterizedTest + void testValidateSettingName_invalidNames(String name, String expectedMessage) { + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> SettingsServiceBean.validateSettingName(name)); + assertEquals(expectedMessage, exception.getMessage()); + } + } + + @Nested + class ValidateSettingLangTest { + + @ValueSource(strings = {"en", "fr", "de"}) + @ParameterizedTest + void testValidateSettingLang_validLanguage(String language) { + assertDoesNotThrow(() -> SettingsServiceBean.validateSettingLang(language)); + } + + @CsvSource({ + ", 'The language ''null'' is not a valid ISO 639-1 language code.'", + "e, 'The language ''e'' is not a valid ISO 639-1 language code.'", + "xyz, 'The language ''xyz'' is not a valid ISO 639-1 language code.'", + "zz, 'The language ''zz'' is not a valid ISO 639-1 language code.'" + }) + @ParameterizedTest + void testValidateSettingLang_invalidLanguage(String language, String expectedMessage) { + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> SettingsServiceBean.validateSettingLang(language)); + assertEquals(expectedMessage, exception.getMessage()); + } + } + @Nested class ListAllAsJsonTest { From 8fe97546e2830fd688ca58ca65d6e21bfb53e6c9 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 17 Jul 2025 17:14:25 +0200 Subject: [PATCH 224/634] feat: add `validateKeys` method to validate settings keys and update tests #11639 Introduced `validateKeys` method in `SettingsServiceBean` to ensure proper settings key validation, including localized and non-localized cases. Added parameterized test coverage for validation scenarios in `SettingsServiceBeanTest`. --- .../settings/SettingsServiceBean.java | 30 +++++++++++++++- .../settings/SettingsServiceBeanTest.java | 36 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 732b2ffad49..4050b6f9e20 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -1124,7 +1124,35 @@ public Set getConfiguredLanguages() { langs.addAll(configuredLocales.keySet()); return langs; } - + + /** + * Validates the keys in the provided settings JSON object. + * This method checks if each key follows the required format and rules. + * If a key is invalid, it is added to the list of invalid keys. + * + * @param settings the JsonObject containing the keys to be validated + * @return a list of invalid keys as an unmodifiable list + */ + public static List validateKeys(JsonObject settings) { + List invalidKeys = new ArrayList<>(); + for (String key : settings.keySet()) { + try { + // Case A: localized setting, validate setting and language + if (key.contains(L10N_KEY_SEPARATOR)) { + String name = key.substring(0, key.indexOf(L10N_KEY_SEPARATOR)); + String lang = key.substring(key.indexOf(L10N_KEY_SEPARATOR) + L10N_KEY_SEPARATOR.length()); + validateSettingName(name); + validateSettingLang(lang); + // Case B: Simple, non-localized setting name + } else { + validateSettingName(key); + } + } catch (IllegalArgumentException iae) { + invalidKeys.add(key); + } + } + return Collections.unmodifiableList(invalidKeys); + } /** * Validates the provided setting name to ensure it meets the required format. diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java index 3f65e5317a4..f504c668dd8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java @@ -105,6 +105,42 @@ void testValidateSettingLang_invalidLanguage(String language, String expectedMes } } + @Nested + class ValidateKeysTest { + static List validateKeysTestParameters() { + return List.of( + Arguments.of( + Json.createObjectBuilder() + .add(":ApplicationTermsOfUse", "validValue1") + .add(":ApplicationTermsOfUse/lang/en", "validValue2") + .build(), + List.of() + ), + Arguments.of( + Json.createObjectBuilder() + .add(":Invalid:Key", "value1") + .add(":NonExistentKey/lang/fr", "value2") + .build(), + List.of(":Invalid:Key", ":NonExistentKey/lang/fr") + ), + Arguments.of( + Json.createObjectBuilder() + .add(":ApplicationTermsOfUse", "value3") + .add("NoColonKey", "value4") + .build(), + List.of("NoColonKey") + ) + ); + } + + @MethodSource("validateKeysTestParameters") + @ParameterizedTest + void testValidateKeys(JsonObject input, List expectedInvalidKeys) { + List result = SettingsServiceBean.validateKeys(input); + assertEquals(expectedInvalidKeys, result); + } + } + @Nested class ListAllAsJsonTest { From 515472d301c9d32621bcc4e77a8ba73df3c76c06 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 17 Jul 2025 17:16:41 +0200 Subject: [PATCH 225/634] feat: add `SettingsServiceBean.convertJsonToSettings()` method and unit tests #11639 Introduced `convertJsonToSettings` in `SettingsServiceBean` to transform JSON objects into `Setting` instances, supporting language-specific keys. Added comprehensive unit tests to verify functionality with various JSON structures. --- .../settings/SettingsServiceBean.java | 35 ++++++++ .../settings/SettingsServiceBeanTest.java | 79 +++++++++++++++++++ 2 files changed, 114 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 4050b6f9e20..9526f1f5b4a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -1077,6 +1077,41 @@ public JsonObject listAllAsJson() { return response.build(); } + /** + * Converts a JSON object representing settings into a list of Setting objects. + * Each entry in the JSON object is processed to create a Setting instance. + * If the key includes a language (indicated by a separator), the language + * information is extracted and included in the Setting object. + * Note: This method expects a pre-validated JsonObject and will happily create + * nonsense settings for you otherwise. + * + * @param settings a (pre-validated) {@link JsonObject} containing key-value pairs where + * each key represents a setting name (and optionally a language code), + * and each value represents the associated content. + * @return a {@link List} of {@link Setting} objects parsed from the input JSON object. + */ + static List convertJsonToSettings(JsonObject settings) { + return settings.entrySet().stream() + .map(entry -> { + String key = entry.getKey(); + String value = entry.getValue().toString() + // This is necessary to avoid storing the quotes inthe DB when a setting is a simple value. + // JsonValue will escape any JsonString with such quotes. + .replaceFirst("^\"", "") + .replaceFirst("\"$", ""); + + if (key.contains(L10N_KEY_SEPARATOR)) { + // Handle localized settings + String name = key.substring(0, key.indexOf(L10N_KEY_SEPARATOR)); + String lang = key.substring(key.indexOf(L10N_KEY_SEPARATOR) + L10N_KEY_SEPARATOR.length()); + return new Setting(name, lang, value); + } else { + return new Setting(key, value); + } + }) + .collect(Collectors.toList()); + } + public Map getBaseMetadataLanguageMap(Map languageMap, boolean refresh) { if (languageMap == null || refresh) { languageMap = new HashMap(); diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java index f504c668dd8..34d791d8fc3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java @@ -231,4 +231,83 @@ void testListAllAsJson_localizedSettings() { assertEquals("value_fr", result.getString("localizedKey/lang/fr")); } } + + @Nested + class ConvertJsonToSettingsTest { + + @Test + void testConvertJsonToSettings_simpleKeyValues() { + // Given + JsonObject input = Json.createObjectBuilder() + .add(":Key1", "Value1") + .add(":Key2", "123456") + // The REST API endpoint presents a JsonObject, which may have number literals in it. + // Check that we can cope with that. + .add(":Key3", 123456) + .build(); + + // When + List result = SettingsServiceBean.convertJsonToSettings(input); + + // Then + assertEquals(3, result.size()); + assertEquals(new Setting(":Key1", "Value1"), result.get(0)); + assertEquals(new Setting(":Key2", "123456"), result.get(1)); + assertEquals(new Setting(":Key3", "123456"), result.get(2)); + } + + @Test + void testConvertJsonToSettings_localizedKeysWithSimpleValues() { + // Given + JsonObject input = Json.createObjectBuilder() + .add(":LocalizedKey/lang/en", "EnglishValue") + .add(":LocalizedKey/lang/fr", "FrenchValue") + .build(); + + // When + List result = SettingsServiceBean.convertJsonToSettings(input); + + // Then + assertEquals(2, result.size()); + assertEquals(new Setting(":LocalizedKey", "en", "EnglishValue"), result.get(0)); + assertEquals(new Setting(":LocalizedKey", "fr", "FrenchValue"), result.get(1)); + } + + @Test + void testConvertJsonToSettings_emptyJson() { + // Given + JsonObject input = Json.createObjectBuilder().build(); + + // When + List result = SettingsServiceBean.convertJsonToSettings(input); + + // Then + assertEquals(0, result.size()); + } + + @Test + void testConvertJsonToSettings_complexJsonValue() { + // Given + JsonObject input = Json.createObjectBuilder() + .add( + ":MaxFileUploadSizeInBytes", + Json.createObjectBuilder() + .add("default", "2147483648") + .add("fileOne", "4000000000") + .add("s3", "8000000000") + .build()) + .build(); + + // When + List result = SettingsServiceBean.convertJsonToSettings(input); + + // Then + assertEquals(1, result.size()); + assertEquals(new Setting(":MaxFileUploadSizeInBytes", + "{\"default\":\"2147483648\",\"fileOne\":\"4000000000\",\"s3\":\"8000000000\"}"), + result.get(0)); + } + + + } } \ No newline at end of file From a1061115726573a610339667c5096854946a5bc3 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 17 Jul 2025 17:21:12 +0200 Subject: [PATCH 226/634] feat: add `putAllSettings` endpoint to Admin API for bulk settings update #11639 Introduced a new `PUT /api/admin/settings` endpoint to update all settings in bulk with JSON input. Added `setAllFromJson` method and placeholder implementation for `replaceAllSettings` in `SettingsServiceBean`. Validates input structure and ensures atomic updates. --- .../edu/harvard/iq/dataverse/api/Admin.java | 23 ++++++++++ .../settings/SettingsServiceBean.java | 42 +++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index fbab4e529b1..e00a93a826f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -63,6 +63,7 @@ import jakarta.ws.rs.PathParam; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; @@ -211,6 +212,28 @@ public Response listAllSettings() { return ok(settingsSvc.listAllAsJson()); } + @Path("settings") + @PUT + @Consumes(MediaType.APPLICATION_JSON) + @APIResponses({ + @APIResponse(responseCode = "200", description = "All database options successfully updated") + }) + public Response putAllSettings(JsonObject settings) { + try { + // Basic JSON structure validation only + if (settings == null || settings.isEmpty()) { + return error(Response.Status.BAD_REQUEST, "Empty or invalid JSON object"); + } + + // Transfer to domain objects and deeper validation to be handled by the service layer. + settingsSvc.setAllFromJson(settings); + return ok("All database options successfully updated."); + + } catch (IllegalArgumentException iae) { + return error(Response.Status.BAD_REQUEST, iae.getMessage()); + } + } + @Path("settings/{name}") @PUT public Response putSetting(@PathParam("name") String name, String content) { diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 9526f1f5b4a..dbb437810e4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -16,10 +16,12 @@ import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; +import jakarta.transaction.Transactional; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -1077,6 +1079,31 @@ public JsonObject listAllAsJson() { return response.build(); } + /** + * Updates all settings by replacing them with the settings provided in the given JSON object. + * This method validates the keys and values from the JSON object, converts them into + * a list of Setting objects, and performs an atomic update of the internal settings. + * + * @param settings a JsonObject containing the new settings to apply. + * Each key corresponds to a setting name, and each value corresponds + * to its respective value. The keys and values will be validated before + * applying the updates. + * @throws IllegalArgumentException if the JSON object contains invalid keys or invalid settings. + */ + public void setAllFromJson(JsonObject settings) { + // Validate the input + List invalidKeys = validateKeys(settings); + if (!invalidKeys.isEmpty()) { + throw new IllegalArgumentException("Invalid key(s): " + String.join(", ", invalidKeys)); + } + + // Convert JSON to Setting objects + List newSettings = convertJsonToSettings(settings); + + // Perform atomic update (replace all settings) + replaceAllSettings(newSettings); + } + /** * Converts a JSON object representing settings into a list of Setting objects. * Each entry in the JSON object is processed to create a Setting instance. @@ -1112,6 +1139,21 @@ static List convertJsonToSettings(JsonObject settings) { .collect(Collectors.toList()); } + /** + * Replaces all existing settings with a new list of settings within a single transaction. + * The operation is atomic, ensuring that either all changes are applied or none in case of a failure. + * + * @param newSettings the list of new {@link Setting} objects to replace the existing settings. + * Must not be null; an empty list clears all settings. + */ + @Transactional + public void replaceAllSettings(List newSettings) { + // Implementation for atomic replacement + // This would involve clearing existing settings and inserting new ones + // within the same transaction + throw new IllegalStateException("Not yet implemented"); + } + public Map getBaseMetadataLanguageMap(Map languageMap, boolean refresh) { if (languageMap == null || refresh) { languageMap = new HashMap(); From 11c8f8211ea27755635731ecd97082f89ccf7d60 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 18 Jul 2025 13:39:48 +0200 Subject: [PATCH 227/634] style: normalize spacing in `Setting` entity named queries #11639 --- src/main/java/edu/harvard/iq/dataverse/settings/Setting.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java b/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java index e429d685c3e..da548195de7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java @@ -24,10 +24,9 @@ @NamedQuery( name="Setting.findByName", query = "SELECT s FROM Setting s WHERE s.name=:name AND s.lang IS NULL" ), @NamedQuery( name="Setting.deleteByNameAndLang", - query="DELETE FROM Setting s WHERE s.name=:name AND s.lang=:lang"), + query="DELETE FROM Setting s WHERE s.name=:name AND s.lang=:lang"), @NamedQuery( name="Setting.findByNameAndLang", - query = "SELECT s FROM Setting s WHERE s.name=:name AND s.lang=:lang" ) - + query="SELECT s FROM Setting s WHERE s.name=:name AND s.lang=:lang") }) @Entity public class Setting implements Serializable { From 71a403c749079376d2f7046b33524de0a8414f94 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 18 Jul 2025 13:40:46 +0200 Subject: [PATCH 228/634] fix: add unique constraint to `Setting` entity for `name` and `lang` fields #11639 --- src/main/java/edu/harvard/iq/dataverse/settings/Setting.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java b/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java index da548195de7..48477b13a3e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java @@ -9,6 +9,8 @@ import jakarta.persistence.NamedQuery; import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; +import jakarta.persistence.Table; +import jakarta.persistence.UniqueConstraint; /** * A single value in the config of dataverse. @@ -29,6 +31,9 @@ query="SELECT s FROM Setting s WHERE s.name=:name AND s.lang=:lang") }) @Entity +@Table(uniqueConstraints = { + @UniqueConstraint(name = "UC_setting_name_lang", columnNames = {"name", "lang"}), +}) public class Setting implements Serializable { @Id From a9b19740c1ea86724afa6b60e42b2b51372e6a9c Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 18 Jul 2025 13:59:24 +0200 Subject: [PATCH 229/634] refactor: update `Setting` entity and queries to treat empty string as default for `lang` #11639 Replaced `NULL` with empty string for `lang` field in `Setting` entity to align with SQL standards and ensure unique constraint behavior. Updated queries, methods, and added safeguards to handle empty `lang` consistently. Enhanced documentation for clarity. --- .../iq/dataverse/settings/Setting.java | 52 +++++++++++++++---- .../settings/SettingsServiceBean.java | 14 ++++- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java b/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java index 48477b13a3e..9c6b1a20baf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java @@ -18,13 +18,13 @@ */ @NamedQueries({ @NamedQuery( name="Setting.deleteByName", - query="DELETE FROM Setting s WHERE s.name=:name AND s.lang IS NULL"), + query="DELETE FROM Setting s WHERE s.name=:name AND s.lang=''"), @NamedQuery( name="Setting.findAll", query="SELECT s FROM Setting s"), @NamedQuery( name="Setting.findAllWithoutLang", - query="SELECT s FROM Setting s WHERE s.lang IS NULL"), + query="SELECT s FROM Setting s WHERE s.lang=''"), @NamedQuery( name="Setting.findByName", - query = "SELECT s FROM Setting s WHERE s.name=:name AND s.lang IS NULL" ), + query="SELECT s FROM Setting s WHERE s.name=:name AND s.lang=''"), @NamedQuery( name="Setting.deleteByNameAndLang", query="DELETE FROM Setting s WHERE s.name=:name AND s.lang=:lang"), @NamedQuery( name="Setting.findByNameAndLang", @@ -42,9 +42,15 @@ public class Setting implements Serializable { @Column(columnDefinition = "TEXT") private String name; - - @Column(columnDefinition = "TEXT") - private String lang; + + /** + * The default value is an empty string, which indicates no specific language is set. + * Using a NULL value here instead would allow the UNIQUE constraint to fail blocking duplicate settings. + * Allowing multiple null within a UNIQUE constraint is part of the SQL standard, which Postgres follows. + * As it stores ISO codes, 10 chars is good enough (ISO codes are 2-8 chars by spec) + */ + @Column(length = 10, nullable = false) + private String lang = ""; @Column(columnDefinition = "TEXT") private String content; @@ -56,11 +62,21 @@ public Setting(String name, String content) { this.name = name; this.content = content; } - + + /** + * Constructs a new Setting object with the specified name, language, and content. + * + * @param name the name of the setting; must not be null + * @param lang the language of the setting, represented as an ISO code; must not be null; + * may be empty to represent a non-localized setting. + * @param content the content or value associated with this setting + * @throws NullPointerException if the name or lang parameters are null + */ public Setting(String name, String lang, String content) { + Objects.requireNonNull(lang, "Setting lang cannot be null"); this.name = name; - this.content = content; this.lang = lang; + this.content = content; } public String getName() { @@ -78,12 +94,28 @@ public String getContent() { public void setContent(String content) { this.content = content; } - + + /** + * Retrieves the language associated with this Setting instance. + * The language is represented as an ISO code string. + * An empty string indicates that no specific localization is set. + * + * @return the language code of this Setting; never null + */ public String getLang() { return lang; } - + + /** + * Sets the language for this Setting instance. + * The language is represented as a non-null ISO code string. + * An empty string indicates that no specific localization shall be set. + * + * @param lang the language code to set; must not be null + * @throws NullPointerException if the provided lang parameter is null + */ public void setLang(String lang) { + Objects.requireNonNull(lang, "Setting lang cannot be null"); this.lang = lang; } diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index dbb437810e4..a90561f642c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -896,6 +896,9 @@ public String get( String name, String defaultValue ) { } public String get(String name, String lang, String defaultValue ) { + // Database safeguard, as the default is an empty string + if (lang == null) lang = ""; + List tokens = em.createNamedQuery("Setting.findByNameAndLang", Setting.class) .setParameter("name", name ) .setParameter("lang", lang ) @@ -912,6 +915,9 @@ public String getValueForKey( Key key, String defaultValue ) { } public String getValueForKey( Key key, String lang, String defaultValue ) { + // Database safeguard, as the default is an empty string + if (lang == null) lang = ""; + return get( key.toString(), lang, defaultValue ); } @@ -939,6 +945,9 @@ public Setting set( String name, String content ) { } public Setting set( String name, String lang, String content ) { + // Database safeguard, as the default is an empty string + if (lang == null) lang = ""; + Setting s = null; List tokens = em.createNamedQuery("Setting.findByNameAndLang", Setting.class) @@ -1008,6 +1017,9 @@ public void delete( String name ) { } public void delete( String name, String lang ) { + // Database safeguard, as the default is an empty string + if (lang == null) lang = ""; + actionLogSvc.log( new ActionLogRecord(ActionLogRecord.ActionType.Setting, "delete") .setInfo(name)); em.createNamedQuery("Setting.deleteByNameAndLang") @@ -1066,7 +1078,7 @@ public JsonObject listAllAsJson() { // Iterate over all the settings and add them to the response. settings.forEach(setting -> { - String name = setting.getName() + (setting.getLang() == null ? "" : L10N_KEY_SEPARATOR + setting.getLang()); + String name = setting.getName() + (setting.getLang().isEmpty() ? "" : L10N_KEY_SEPARATOR + setting.getLang()); // In case the setting is a JSON object, treat it a such in the output (so the API can return valid JSON) if (setting.getContent().trim().startsWith("{")) From 5ae758c601d10e806159c48e5fbe5d1b68302155 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 18 Jul 2025 14:01:38 +0200 Subject: [PATCH 230/634] refactor: enforce non-null constraint, VARCHAR type, and limit length for `name` in `Setting` entity #11639 Updated `name` field to enforce a maximum length of 200 characters on a VARCHAR field for better performance (replacing the TEXT column type) and mark it as non-nullable (null settings make no sense at all). Added `Objects.requireNonNull` validation for `name` in entity methods and constructors to ensure consistent behavior. --- .../java/edu/harvard/iq/dataverse/settings/Setting.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java b/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java index 9c6b1a20baf..aa9f6559e35 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java @@ -40,7 +40,7 @@ public class Setting implements Serializable { @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; - @Column(columnDefinition = "TEXT") + @Column(length = 200, nullable = false) private String name; /** @@ -59,8 +59,9 @@ public Setting() { } public Setting(String name, String content) { - this.name = name; - this.content = content; + Objects.requireNonNull(name, "Setting name cannot be null"); + this.name = name; + this.content = content; } /** @@ -73,6 +74,7 @@ public Setting(String name, String content) { * @throws NullPointerException if the name or lang parameters are null */ public Setting(String name, String lang, String content) { + Objects.requireNonNull(name, "Setting name cannot be null"); Objects.requireNonNull(lang, "Setting lang cannot be null"); this.name = name; this.lang = lang; @@ -84,6 +86,7 @@ public String getName() { } public void setName(String name) { + Objects.requireNonNull(name, "Setting name cannot be null"); this.name = name; } From 974d6c4c1634dbd6fefb21838b40646ebc913ac0 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 18 Jul 2025 14:01:49 +0200 Subject: [PATCH 231/634] chore: update migration script comment for clarity #11639 --- src/main/resources/db/migration/V6.7.0.1.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V6.7.0.1.sql b/src/main/resources/db/migration/V6.7.0.1.sql index 733a10d951f..aa3683873ca 100644 --- a/src/main/resources/db/migration/V6.7.0.1.sql +++ b/src/main/resources/db/migration/V6.7.0.1.sql @@ -1,4 +1,4 @@ --- This script migrates the old TabularIngestSizeLimit database setting using format suffixes to a JSON based approach. +-- Migrates the old TabularIngestSizeLimit database setting using format suffixes to a JSON based approach. See #11639 DO $$ DECLARE From c3f1ed7de4b99a79c67cd2840bc1cf1643f3e5bd Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 18 Jul 2025 14:02:34 +0200 Subject: [PATCH 232/634] chore: add migration script to update `Setting` table structure #11639 Introduced SQL migration to optimize `Setting` table by switching `TEXT` columns to `VARCHAR` for better performance, enforcing `NOT NULL` constraints, adding a unique constraint for `name` and `lang`, and setting default empty string for `lang`. Includes logic to handle existing data and conditional checks for schema changes. --- src/main/resources/db/migration/V6.7.0.2.sql | 35 ++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 src/main/resources/db/migration/V6.7.0.2.sql diff --git a/src/main/resources/db/migration/V6.7.0.2.sql b/src/main/resources/db/migration/V6.7.0.2.sql new file mode 100644 index 00000000000..177f88332b7 --- /dev/null +++ b/src/main/resources/db/migration/V6.7.0.2.sql @@ -0,0 +1,35 @@ +-- Update Setting table structure for changes from #11639 +-- 1. Change column types from TEXT to VARCHAR for better performance +-- 2. Update lang column to use empty string default instead of NULL (avoid non-unique pairs) +-- 3. Add NOT NULL constraints and unique constraint for name+lang pairs + +-- First, update any existing NULL lang values to empty string +UPDATE Setting SET lang = '' WHERE lang IS NULL; + +-- Postgres doesn't support IF NOT EXISTS for ALTER COLUMN or ADD CONSTRAINT, so we need conditional logic +DO $$ +BEGIN + -- Only alter columns if they need to be changed + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'Setting' AND column_name = 'name' + AND (data_type = 'text' OR is_nullable = 'YES')) THEN + ALTER TABLE setting ALTER COLUMN name TYPE VARCHAR(255); + ALTER TABLE Setting ALTER COLUMN name SET NOT NULL; + END IF; + + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'Setting' AND column_name = 'lang' + AND (data_type = 'text' OR is_nullable = 'YES')) THEN + ALTER TABLE Setting ALTER COLUMN lang TYPE VARCHAR(10); + ALTER TABLE Setting ALTER COLUMN lang SET NOT NULL; + ALTER TABLE Setting ALTER COLUMN lang SET DEFAULT ''; + END IF; + + IF NOT EXISTS (SELECT 1 FROM information_schema.table_constraints + WHERE table_name = 'Setting' + AND constraint_name = 'UC_setting_name_lang' + AND constraint_type = 'UNIQUE') THEN + ALTER TABLE Setting ADD CONSTRAINT UC_setting_name_lang UNIQUE (name, lang); + END IF; + +END $$; From 782542f7291e25950afeba62496a726eced8327f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 18 Jul 2025 14:03:42 +0200 Subject: [PATCH 233/634] refactor: enhance `Setting` entity equality and constructor for clarity and robustness #11639 Refined `equals` and `hashCode` methods to base equality comparison on `name` and `lang` fields, improving consistency and alignment with usage patterns. Updated the no-argument constructor to protected, adding explanatory comments for enforced design intent and JPA compatibility. --- .../iq/dataverse/settings/Setting.java | 37 ++++++++++++------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java b/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java index aa9f6559e35..0f240302366 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java @@ -55,7 +55,9 @@ public class Setting implements Serializable { @Column(columnDefinition = "TEXT") private String content; - public Setting() { + protected Setting() { + // Intentionally left blank - no empty settings allowed. + // Protected visibility to allow JPA to work. } public Setting(String name, String content) { @@ -124,26 +126,33 @@ public void setLang(String lang) { @Override public int hashCode() { - int hash = 7; - hash = 73 * hash + Objects.hashCode(this.name); - return hash; + return Objects.hash(name, lang); } - + + /** + * Compares this Setting instance to another object for equality. Two Setting + * objects are considered equal if their {@code name} and {@code lang} fields are + * both equal. + * @implNote We do not use the {@code id} and {@code content} fields for the comparison. + * This is due to how these objects usually are used: + * - Mutable content to use for comparison may break collections. + * - Configuration management requires stable identity based on setting's name and localization. + * The content of the settings is irrelevant for lookups. + * + * @param obj the object to compare this Setting with + * @return {@code true} if the specified object is equal to this Setting, {@code false} otherwise + */ @Override public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if ( !(obj instanceof Setting) ) { - return false; + if (this == obj) { + return true; } - final Setting other = (Setting) obj; - if (!Objects.equals(this.name, other.name)) { + if (!(obj instanceof Setting other)) { return false; } - return Objects.equals(this.content, other.content); + return Objects.equals(this.name, other.name) && Objects.equals(this.lang, other.lang); } - + @Override public String toString() { return "[Setting name:" + getName() + " value:" + getContent() + "]"; From aed9f360987cc4aef0198be61ca7c933f2ed49f4 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 18 Jul 2025 15:24:56 +0200 Subject: [PATCH 234/634] refactor: rename `BuiltinUsers.KEY` setting to `:BuiltinUsersKey` for alignment and consistency #11639 Updated references, documentation, and added migration script to reflect the renaming and ensure adherence to naming conventions. Marked the setting as deprecated for removal. --- doc/sphinx-guides/source/api/native-api.rst | 4 ++-- doc/sphinx-guides/source/developers/testing.rst | 2 +- doc/sphinx-guides/source/installation/config.rst | 12 +++++++----- .../scripts/bootstrap/demo/init.sh | 2 +- scripts/api/post-install-api-block.sh | 2 +- scripts/api/setup-all.sh | 4 ++-- scripts/api/setup-users.sh | 2 +- scripts/issues/2454/run-test.sh | 2 +- .../edu/harvard/iq/dataverse/api/BuiltinUsers.java | 4 +--- .../iq/dataverse/settings/SettingsServiceBean.java | 9 +++++++++ src/main/resources/db/migration/V6.7.0.1.sql | 14 ++++++++++++-- 11 files changed, 38 insertions(+), 19 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index fa4b4611559..7dc50e7a532 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5485,13 +5485,13 @@ Builtin users are known as "Username/Email and Password" users in the :doc:`/use Create a Builtin User ~~~~~~~~~~~~~~~~~~~~~ -For security reasons, builtin users cannot be created via API unless the team who runs the Dataverse installation has populated a database setting called ``BuiltinUsers.KEY``, which is described under :ref:`securing-your-installation` and :ref:`database-settings` sections of Configuration in the Installation Guide. You will need to know the value of ``BuiltinUsers.KEY`` before you can proceed. +For security reasons, builtin users cannot be created via API unless the team who runs the Dataverse installation has populated a database setting called ``:BuiltinUsersKey``, which is described under :ref:`securing-your-installation` and :ref:`database-settings` sections of Configuration in the Installation Guide. You will need to know the value of ``:BuiltinUsersKey`` before you can proceed. To create a builtin user via API, you must first construct a JSON document. You can download :download:`user-add.json <../_static/api/user-add.json>` or copy the text below as a starting point and edit as necessary. .. literalinclude:: ../_static/api/user-add.json -Place this ``user-add.json`` file in your current directory and run the following curl command, substituting variables as necessary. Note that both the password of the new user and the value of ``BuiltinUsers.KEY`` are passed as query parameters:: +Place this ``user-add.json`` file in your current directory and run the following curl command, substituting variables as necessary. Note that both the password of the new user and the value of ``:BuiltinUsersKey`` are passed as query parameters:: curl -d @user-add.json -H "Content-type:application/json" "$SERVER_URL/api/builtin-users?password=$NEWUSER_PASSWORD&key=$BUILTIN_USERS_KEY" diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index 1690864d453..fa279b3dbab 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -209,7 +209,7 @@ The Burrito Key For reasons that have been lost to the mists of time, the Dataverse software really wants you to to have a burrito. Specifically, if you're trying to run REST Assured tests and see the error "Dataverse config issue: No API key defined for built in user management", you must run the following curl command (or make an equivalent change to your database): -``curl -X PUT -d 'burrito' http://localhost:8080/api/admin/settings/BuiltinUsers.KEY`` +``curl -X PUT -d 'burrito' http://localhost:8080/api/admin/settings/:BuiltinUsersKey`` Without this "burrito" key in place, REST Assured will not be able to create users. We create users to create objects we want to test, such as collections, datasets, and files. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 0541f9f301d..8722da19b78 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -25,7 +25,7 @@ The default password for the "dataverseAdmin" superuser account is "admin", as m Blocking API Endpoints ++++++++++++++++++++++ -The :doc:`/api/native-api` contains a useful but potentially dangerous set of API endpoints called "admin" that allows you to change system settings, make ordinary users into superusers, and more. The "builtin-users" endpoints let admins do tasks such as creating a local/builtin user account if they know the key defined in :ref:`BuiltinUsers.KEY`. +The :doc:`/api/native-api` contains a useful but potentially dangerous set of API endpoints called "admin" that allows you to change system settings, make ordinary users into superusers, and more. The "builtin-users" endpoints let admins do tasks such as creating a local/builtin user account if they know the key defined in :ref:`:BuiltinUsersKey`. By default in the code, most of these API endpoints can be operated on remotely and a number of endpoints do not require authentication. However, the endpoints "admin" and "builtin-users" are limited to localhost out of the box by the installer, using the JvmSettings :ref:`dataverse.api.blocked.endpoints` and :ref:`dataverse.api.blocked.policy`. @@ -781,7 +781,7 @@ Both Local and Remote Auth The ``authenticationproviderrow`` database table controls which "authentication providers" are available within a Dataverse installation. Out of the box, a single row with an id of "builtin" will be present. For each user in a Dataverse installation, the ``authenticateduserlookup`` table will have a value under ``authenticationproviderid`` that matches this id. For example, the default "dataverseAdmin" user will have the value "builtin" under ``authenticationproviderid``. Why is this important? Users are tied to a specific authentication provider but conversion mechanisms are available to switch a user from one authentication provider to the other. As explained in the :doc:`/user/account` section of the User Guide, a graphical workflow is provided for end users to convert from the "builtin" authentication provider to a remote provider. Conversion from a remote authentication provider to the builtin provider can be performed by a sysadmin with access to the "admin" API. See the :doc:`/api/native-api` section of the API Guide for how to list users and authentication providers as JSON. -Adding and enabling a second authentication provider (:ref:`native-api-add-auth-provider` and :ref:`api-toggle-auth-provider`) will result in the Log In page showing additional providers for your users to choose from. By default, the Log In page will show the "builtin" provider, but you can adjust this via the :ref:`conf-default-auth-provider` configuration option. Further customization can be achieved by setting :ref:`conf-allow-signup` to "false", thus preventing users from creating local accounts via the web interface. Please note that local accounts can also be created through the API by enabling the ``builtin-users`` endpoint (:ref:`:BlockedApiEndpoints`) and setting the ``BuiltinUsers.KEY`` database setting (:ref:`BuiltinUsers.KEY`). +Adding and enabling a second authentication provider (:ref:`native-api-add-auth-provider` and :ref:`api-toggle-auth-provider`) will result in the Log In page showing additional providers for your users to choose from. By default, the Log In page will show the "builtin" provider, but you can adjust this via the :ref:`conf-default-auth-provider` configuration option. Further customization can be achieved by setting :ref:`conf-allow-signup` to "false", thus preventing users from creating local accounts via the web interface. Please note that local accounts can also be created through the API by enabling the ``builtin-users`` endpoint (:ref:`:BlockedApiEndpoints`) and setting the ``:BuiltinUsersKey`` database setting (:ref:`:BuiltinUsersKey`). To configure Shibboleth see the :doc:`shibboleth` section and to configure OAuth see the :doc:`oauth2` section. @@ -3933,14 +3933,16 @@ Now that ``:BlockedApiKey`` has been enabled, blocked APIs can be accessed using ``curl https://demo.dataverse.org/api/admin/settings?unblock-key=theKeyYouChose`` -.. _BuiltinUsers.KEY: +.. _:BuiltinUsersKey: -BuiltinUsers.KEY +:BuiltinUsersKey ++++++++++++++++ The key required to create users via API as documented at :doc:`/api/native-api`. Unlike other database settings, this one doesn't start with a colon. -``curl -X PUT -d builtInS3kretKey http://localhost:8080/api/admin/settings/BuiltinUsers.KEY`` +``curl -X PUT -d builtInS3kretKey http://localhost:8080/api/admin/settings/:BuiltinUsersKey`` + +Note: this key used to be named ``BuiltinUsers.KEY`` until Dataverse 6.8. :SearchApiRequiresToken +++++++++++++++++++++++ diff --git a/modules/container-configbaker/scripts/bootstrap/demo/init.sh b/modules/container-configbaker/scripts/bootstrap/demo/init.sh index aa73cb5edff..b2735b50b28 100644 --- a/modules/container-configbaker/scripts/bootstrap/demo/init.sh +++ b/modules/container-configbaker/scripts/bootstrap/demo/init.sh @@ -31,7 +31,7 @@ fi echo "" echo "Revoke the key that allows for creation of builtin users..." -curl -sS -X DELETE "${DATAVERSE_URL}/api/admin/settings/BuiltinUsers.KEY" +curl -sS -X DELETE "${DATAVERSE_URL}/api/admin/settings/:BuiltinUsersKey" # TODO: stop using these deprecated database settings. See https://github.com/IQSS/dataverse/pull/11454 echo "" diff --git a/scripts/api/post-install-api-block.sh b/scripts/api/post-install-api-block.sh index 4cc0ac783f7..f7753665b5b 100755 --- a/scripts/api/post-install-api-block.sh +++ b/scripts/api/post-install-api-block.sh @@ -4,7 +4,7 @@ # the sensitive API endpoints, in order to block it for the general public. # First, revoke the authentication token from the built-in user: -curl -X DELETE $SERVER/admin/settings/BuiltinUsers.KEY +curl -X DELETE "$SERVER/admin/settings/:BuiltinUsersKey" # Block the sensitive endpoints: # Relevant settings: diff --git a/scripts/api/setup-all.sh b/scripts/api/setup-all.sh index b7f962209e4..bd0bd77c52b 100755 --- a/scripts/api/setup-all.sh +++ b/scripts/api/setup-all.sh @@ -57,7 +57,7 @@ echo "- Allow internal signup" curl -X PUT -d yes "${DATAVERSE_URL}/api/admin/settings/:AllowSignUp" curl -X PUT -d "/dataverseuser.xhtml?editMode=CREATE" "${DATAVERSE_URL}/api/admin/settings/:SignUpUrl" -curl -X PUT -d burrito "${DATAVERSE_URL}/api/admin/settings/BuiltinUsers.KEY" +curl -X PUT -d burrito "${DATAVERSE_URL}/api/admin/settings/:BuiltinUsersKey" curl -X PUT -d localhost-only "${DATAVERSE_URL}/api/admin/settings/:BlockedApiPolicy" curl -X PUT -d 'native/http' "${DATAVERSE_URL}/api/admin/settings/:UploadMethods" echo @@ -91,7 +91,7 @@ if [ $SECURESETUP = 1 ] then # Revoke the "burrito" super-key; # Block sensitive API endpoints; - curl -X DELETE "${DATAVERSE_URL}/api/admin/settings/BuiltinUsers.KEY" + curl -X DELETE "${DATAVERSE_URL}/api/admin/settings/:BuiltinUsersKey" curl -X PUT -d 'admin,builtin-users' "${DATAVERSE_URL}/api/admin/settings/:BlockedApiEndpoints" echo "Access to the /api/admin and /api/test is now disabled, except for connections from localhost." else diff --git a/scripts/api/setup-users.sh b/scripts/api/setup-users.sh index 141e1b3150f..7df771dc0fe 100755 --- a/scripts/api/setup-users.sh +++ b/scripts/api/setup-users.sh @@ -5,7 +5,7 @@ SERVER=http://localhost:8080/api echo Setting up users on $SERVER echo ============================================== -curl -X PUT -d burrito $SERVER/admin/settings/BuiltinUsers.KEY +curl -X PUT -d burrito "$SERVER/admin/settings/:BuiltinUsersKey" peteResp=$(curl -s -H "Content-type:application/json" -X POST -d @data/userPete.json "$SERVER/builtin-users?password=pete&key=burrito") diff --git a/scripts/issues/2454/run-test.sh b/scripts/issues/2454/run-test.sh index 49eb45a8a5e..5ae0ac33f4d 100755 --- a/scripts/issues/2454/run-test.sh +++ b/scripts/issues/2454/run-test.sh @@ -39,7 +39,7 @@ if [ $SETUP_NEEDED == "yes" ]; then echo $ROOT_USER api key is $ROOT_KEY # Create @anAuthUser - USER_CREATION_KEY=$($DB "SELECT content FROM setting WHERE name='BuiltinUsers.KEY'") + USER_CREATION_KEY=$($DB "SELECT content FROM setting WHERE name=':BuiltinUsersKey'") AN_AUTH_USER_KEY=$( curl -s -X POST -d@anAuthUser.json -H"Content-type:application/json" $ENDPOINT/builtin-users?password=XXX\&key=$USER_CREATION_KEY | jq .data.apiToken | tr -d \") ANOTHER_AUTH_USER_KEY=$( curl -s -X POST -d@anotherAuthUser.json -H"Content-type:application/json" $ENDPOINT/builtin-users?password=XXX\&key=$USER_CREATION_KEY | jq .data.apiToken | tr -d \") echo diff --git a/src/main/java/edu/harvard/iq/dataverse/api/BuiltinUsers.java b/src/main/java/edu/harvard/iq/dataverse/api/BuiltinUsers.java index 317f7d6c870..79d5682d4f3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/BuiltinUsers.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/BuiltinUsers.java @@ -40,8 +40,6 @@ public class BuiltinUsers extends AbstractApiBean { private static final Logger logger = Logger.getLogger(BuiltinUsers.class.getName()); - private static final String API_KEY_IN_SETTINGS = "BuiltinUsers.KEY"; - @EJB protected BuiltinUserServiceBean builtinUserSvc; @@ -129,7 +127,7 @@ private Response internalSave(BuiltinUser user, String password, String key) { } private Response internalSave(BuiltinUser user, String password, String key, Boolean sendEmailNotification) { - String expectedKey = settingsSvc.get(API_KEY_IN_SETTINGS); + String expectedKey = settingsSvc.getValueForKey(SettingsServiceBean.Key.BuiltinUsersKey); if (expectedKey == null) { return error(Status.SERVICE_UNAVAILABLE, "Dataverse config issue: No API key defined for built in user management"); diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index a90561f642c..e9275792129 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -166,6 +166,15 @@ public enum Key { @Deprecated(forRemoval = true, since = "2025-04-29") BlockedApiPolicy, + /** + * A special secret that, if set, needs to be given when trying to manage internal users. + * This key was formerly known as "BuiltinUsers.KEY", which never was a setting name aligning with the others. + * At some future point this setting should be moved to JvmSettings (so we consume proper secrets) + * or plainly removed with the transition to the SPA frontend requiring an external IdP. + */ + @Deprecated(forRemoval = true, since = "2025-08-01") + BuiltinUsersKey, + /** * For development only (see dev guide for details). Backed by an enum * of possible account types. diff --git a/src/main/resources/db/migration/V6.7.0.1.sql b/src/main/resources/db/migration/V6.7.0.1.sql index aa3683873ca..cfe1e75d0b5 100644 --- a/src/main/resources/db/migration/V6.7.0.1.sql +++ b/src/main/resources/db/migration/V6.7.0.1.sql @@ -1,5 +1,6 @@ --- Migrates the old TabularIngestSizeLimit database setting using format suffixes to a JSON based approach. See #11639 - +-- Migrates the old database setting to their valid and aligned successors. #11639 +-- 1. ":TabularIngestSizeLimit" database setting used format suffixes, move to a JSON-based approach +-- 2. (see below) "BuiltinUsers.KEY" was never aligned with any of the other settings names. DO $$ DECLARE base_setting_content TEXT; @@ -75,4 +76,13 @@ DO $$ AND lang IS NULL; RAISE NOTICE 'Successfully migrated TabularIngestSizeLimit settings to JSON format: %', json_object::TEXT; + END $$; + +-- 2. Migrate BuiltinUsers.KEY to the new setting name +DO $$ + BEGIN + IF EXISTS (SELECT 1 FROM Setting WHERE name = 'BuiltinUsers.KEY') THEN + INSERT INTO Setting (name, lang, content) VALUES (':BuiltinUsersKey', NULL, (SELECT content FROM Setting WHERE name = 'BuiltinUsers.KEY')); + DELETE FROM Setting WHERE name = 'BuiltinUsers.KEY'; + END IF; END $$; \ No newline at end of file From 48db4b39dde54e2161ac89a89c4828d4295bd9cd Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 18 Jul 2025 17:28:32 +0200 Subject: [PATCH 235/634] refactor: extract `convertToJsonKey` method in `SettingsServiceBean` for reusability #11639 Moved logic for constructing JSON keys into a dedicated method `convertToJsonKey`, improving code clarity and reducing duplication. --- .../harvard/iq/dataverse/settings/SettingsServiceBean.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index e9275792129..78ffb92c819 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -1087,7 +1087,7 @@ public JsonObject listAllAsJson() { // Iterate over all the settings and add them to the response. settings.forEach(setting -> { - String name = setting.getName() + (setting.getLang().isEmpty() ? "" : L10N_KEY_SEPARATOR + setting.getLang()); + String name = convertToJsonKey(setting); // In case the setting is a JSON object, treat it a such in the output (so the API can return valid JSON) if (setting.getContent().trim().startsWith("{")) @@ -1223,6 +1223,10 @@ public Set getConfiguredLanguages() { return langs; } + public static String convertToJsonKey(Setting setting) { + return setting.getName() + (setting.getLang().isEmpty() ? "" : L10N_KEY_SEPARATOR + setting.getLang()); + } + /** * Validates the keys in the provided settings JSON object. * This method checks if each key follows the required format and rules. From a736052069973f83d7a126bbbafe61940231363f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 18 Jul 2025 17:30:21 +0200 Subject: [PATCH 236/634] feat: implement atomic bulk settings replacement with detailed operation tracking in `SettingsServiceBean` #11639 Added `replaceAllSettings` method to support atomic replacement of settings with clear distinctions for created, updated, and deleted items. Enhanced clarity by introducing an `Op` enum for operation types and a `convertToJson` helper method to generate JSON representations of changes. --- .../settings/SettingsServiceBean.java | 112 ++++++++++++++++-- 1 file changed, 103 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 78ffb92c819..e0bab4dadc9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -10,6 +10,7 @@ import jakarta.inject.Named; import jakarta.json.Json; import jakarta.json.JsonArray; +import jakarta.json.JsonArrayBuilder; import jakarta.json.JsonObject; import jakarta.json.JsonObjectBuilder; import jakarta.json.JsonValue; @@ -32,6 +33,7 @@ import java.util.Map; import java.util.Set; import java.util.StringTokenizer; +import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -1161,18 +1163,110 @@ static List convertJsonToSettings(JsonObject settings) { } /** - * Replaces all existing settings with a new list of settings within a single transaction. - * The operation is atomic, ensuring that either all changes are applied or none in case of a failure. + * Enum representing the types of operations that are performed on a bulk operation with settings. + */ + static enum Op { + UPDATED, + CREATED, + DELETED; + + static JsonObjectBuilder convertToJson(Map operationalDetails) { + // Create a nice represenation of what happened as Json + JsonObjectBuilder jbo = Json.createObjectBuilder(); + JsonArrayBuilder created = Json.createArrayBuilder(); + JsonArrayBuilder updated = Json.createArrayBuilder(); + JsonArrayBuilder deleted = Json.createArrayBuilder(); + + operationalDetails.forEach((setting, op) -> { + String name = convertToJsonKey(setting); + switch (op) { + case CREATED -> created.add(name); + case UPDATED -> updated.add(name); + case DELETED -> deleted.add(name); + } + }); + + return jbo + .add("created", created) + .add("updated", updated) + .add("deleted", deleted); + } + } + + /** + * Replaces all existing settings in the database with the provided set of new settings. + * This method performs the following actions: + * - Deletes any existing settings that are not present in the provided new settings. + * - Updates the content of existing settings that match the keys in the provided new settings. + * - Creates new settings that are not present in the database. * - * @param newSettings the list of new {@link Setting} objects to replace the existing settings. - * Must not be null; an empty list clears all settings. + * @param newSettings the set of new settings to replace the existing ones. + * Each setting is uniquely identified by its name and language. + * Must not be null (it may be empty). + * @return a map tracking the operations performed on each setting. The map's keys + * are the settings involved, and the values are the types of operations + * performed (CREATED, UPDATED, DELETED). */ @Transactional - public void replaceAllSettings(List newSettings) { - // Implementation for atomic replacement - // This would involve clearing existing settings and inserting new ones - // within the same transaction - throw new IllegalStateException("Not yet implemented"); + public Map replaceAllSettings(Set newSettings) { + Objects.requireNonNull(newSettings, "The list of new settings cannot be null (it may be empty)."); + + // Get all existing settings as a map for O(1) lookup + List existingSettings = em.createNamedQuery("Setting.findAll", Setting.class).getResultList(); + Map existingByKey = existingSettings.stream() + .collect(Collectors.toMap( + setting -> setting.getName() + "|" + setting.getLang(), + Function.identity() + )); + + // Create map of new settings for O(1) lookup + Map newByKey = newSettings.stream() + .collect(Collectors.toMap( + setting -> setting.getName() + "|" + setting.getLang(), + Function.identity() + )); + + // Track operations for return value + Map opsTracking = new HashMap<>(); + + // Process existing settings + for (Map.Entry entry : existingByKey.entrySet()) { + String key = entry.getKey(); + Setting existingSetting = entry.getValue(); + + // Setting exists in DB but not in new set - delete it + if (!newByKey.containsKey(key)) { + em.remove(existingSetting); + opsTracking.put(existingSetting, Op.DELETED); + + // Setting exists in both - update with new values + } else { + Setting newSetting = newByKey.get(key); + // We use the already managed entity and update it with the content of the new setting. + // (This means we don't need to call em.merge(), the ORM will track and execute it for us.) + existingSetting.setContent(newSetting.getContent()); + opsTracking.put(existingSetting, Op.UPDATED); + } + } + + // Process new settings - create those not in existing set + for (Map.Entry entry : newByKey.entrySet()) { + String key = entry.getKey(); + Setting newSetting = entry.getValue(); + + if (!existingByKey.containsKey(key)) { + // Setting is new - persist it + em.persist(newSetting); + opsTracking.put(newSetting, Op.CREATED); + } + // If it exists, it was already handled in the previous loop + } + + // Flush changes to ensure consistency before transaction is committed (will also ensure merge() is called). + em.flush(); + + return opsTracking; + } public Map getBaseMetadataLanguageMap(Map languageMap, boolean refresh) { From c68ac57f9cd0f7ce232e40f154dab54d5d920e0e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 18 Jul 2025 17:32:32 +0200 Subject: [PATCH 237/634] refactor: change `List` to `Set` for `convertJsonToSettings` #11639 - enforce unique `Setting` objects - enable easier detection of differences between two sets if settings --- .../settings/SettingsServiceBean.java | 9 ++++--- .../settings/SettingsServiceBeanTest.java | 25 +++++++++++-------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index e0bab4dadc9..f30814a2a4b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -1121,7 +1121,7 @@ public void setAllFromJson(JsonObject settings) { } // Convert JSON to Setting objects - List newSettings = convertJsonToSettings(settings); + Set newSettings = convertJsonToSettings(settings); // Perform atomic update (replace all settings) replaceAllSettings(newSettings); @@ -1133,14 +1133,15 @@ public void setAllFromJson(JsonObject settings) { * If the key includes a language (indicated by a separator), the language * information is extracted and included in the Setting object. * Note: This method expects a pre-validated JsonObject and will happily create - * nonsense settings for you otherwise. + * nonsense settings for you otherwise. This is a reason for the package visibility. * * @param settings a (pre-validated) {@link JsonObject} containing key-value pairs where * each key represents a setting name (and optionally a language code), * and each value represents the associated content. * @return a {@link List} of {@link Setting} objects parsed from the input JSON object. */ - static List convertJsonToSettings(JsonObject settings) { + static Set convertJsonToSettings(JsonObject settings) { + Objects.requireNonNull(settings, "The settings object cannot be null."); return settings.entrySet().stream() .map(entry -> { String key = entry.getKey(); @@ -1159,7 +1160,7 @@ static List convertJsonToSettings(JsonObject settings) { return new Setting(key, value); } }) - .collect(Collectors.toList()); + .collect(Collectors.toSet()); } /** diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java index 34d791d8fc3..bcc82933226 100644 --- a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java @@ -16,6 +16,7 @@ import java.util.Collections; import java.util.List; +import java.util.Set; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -247,13 +248,15 @@ void testConvertJsonToSettings_simpleKeyValues() { .build(); // When - List result = SettingsServiceBean.convertJsonToSettings(input); + Set result = SettingsServiceBean.convertJsonToSettings(input); // Then assertEquals(3, result.size()); - assertEquals(new Setting(":Key1", "Value1"), result.get(0)); - assertEquals(new Setting(":Key2", "123456"), result.get(1)); - assertEquals(new Setting(":Key3", "123456"), result.get(2)); + assertEquals( + Set.of(new Setting(":Key1", "Value1"), + new Setting(":Key2", "123456"), + new Setting(":Key3", "123456") + ), result); } @Test @@ -265,12 +268,14 @@ void testConvertJsonToSettings_localizedKeysWithSimpleValues() { .build(); // When - List result = SettingsServiceBean.convertJsonToSettings(input); + Set result = SettingsServiceBean.convertJsonToSettings(input); // Then assertEquals(2, result.size()); - assertEquals(new Setting(":LocalizedKey", "en", "EnglishValue"), result.get(0)); - assertEquals(new Setting(":LocalizedKey", "fr", "FrenchValue"), result.get(1)); + assertEquals( + Set.of(new Setting(":LocalizedKey", "en", "EnglishValue"), + new Setting(":LocalizedKey", "fr", "FrenchValue") + ), result); } @Test @@ -279,7 +284,7 @@ void testConvertJsonToSettings_emptyJson() { JsonObject input = Json.createObjectBuilder().build(); // When - List result = SettingsServiceBean.convertJsonToSettings(input); + Set result = SettingsServiceBean.convertJsonToSettings(input); // Then assertEquals(0, result.size()); @@ -299,13 +304,13 @@ void testConvertJsonToSettings_complexJsonValue() { .build(); // When - List result = SettingsServiceBean.convertJsonToSettings(input); + Set result = SettingsServiceBean.convertJsonToSettings(input); // Then assertEquals(1, result.size()); assertEquals(new Setting(":MaxFileUploadSizeInBytes", "{\"default\":\"2147483648\",\"fileOne\":\"4000000000\",\"s3\":\"8000000000\"}"), - result.get(0)); + result.stream().toList().get(0)); } From 6f90535c0bfb03aca7d2290344e701775dbf3304 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 18 Jul 2025 17:33:44 +0200 Subject: [PATCH 238/634] fix: add null checks for `settings` in `setAllFromJson` and `validateKeys` #11639 --- .../harvard/iq/dataverse/settings/SettingsServiceBean.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index f30814a2a4b..e40165668d6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.StringTokenizer; import java.util.function.Function; @@ -1114,6 +1115,10 @@ public JsonObject listAllAsJson() { * @throws IllegalArgumentException if the JSON object contains invalid keys or invalid settings. */ public void setAllFromJson(JsonObject settings) { + if (settings == null) { + throw new IllegalArgumentException("Settings cannot be null"); + } + // Validate the input List invalidKeys = validateKeys(settings); if (!invalidKeys.isEmpty()) { @@ -1331,6 +1336,7 @@ public static String convertToJsonKey(Setting setting) { * @return a list of invalid keys as an unmodifiable list */ public static List validateKeys(JsonObject settings) { + Objects.requireNonNull(settings, "The settings object cannot be null."); List invalidKeys = new ArrayList<>(); for (String key : settings.keySet()) { try { From 1f8a280711bf37c05aa3273c5f6302d1dd8a4b67 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 18 Jul 2025 17:38:30 +0200 Subject: [PATCH 239/634] feat: return operation details from `setAllFromJson` in `SettingsServiceBean` after bulk operation #11639 --- .../edu/harvard/iq/dataverse/api/Admin.java | 4 ++-- .../settings/SettingsServiceBean.java | 23 ++++++++++--------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index e00a93a826f..1dc6e70fa3f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -226,8 +226,8 @@ public Response putAllSettings(JsonObject settings) { } // Transfer to domain objects and deeper validation to be handled by the service layer. - settingsSvc.setAllFromJson(settings); - return ok("All database options successfully updated."); + JsonObjectBuilder successfullOperations = settingsSvc.setAllFromJson(settings); + return ok("All database options successfully updated.", successfullOperations); } catch (IllegalArgumentException iae) { return error(Response.Status.BAD_REQUEST, iae.getMessage()); diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index e40165668d6..cd497d91329 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -1104,17 +1104,16 @@ public JsonObject listAllAsJson() { } /** - * Updates all settings by replacing them with the settings provided in the given JSON object. - * This method validates the keys and values from the JSON object, converts them into - * a list of Setting objects, and performs an atomic update of the internal settings. + * Updates all current settings from the specified JSON object. Validates the input JSON, + * converts it to a set of settings, and replaces all existing settings with the new ones + * in an atomic operation. If the settings object is null, contains invalid keys, or if the new + * set of settings is empty, the method throws an appropriate exception. * - * @param settings a JsonObject containing the new settings to apply. - * Each key corresponds to a setting name, and each value corresponds - * to its respective value. The keys and values will be validated before - * applying the updates. - * @throws IllegalArgumentException if the JSON object contains invalid keys or invalid settings. + * @param settings the JSON object containing the new configuration settings to be applied; must not be null + * @return a JsonObjectBuilder representing the operational details of the applied updates + * @throws IllegalArgumentException if the settings object is null, contains invalid keys, or results in empty settings */ - public void setAllFromJson(JsonObject settings) { + public JsonObjectBuilder setAllFromJson(JsonObject settings) { if (settings == null) { throw new IllegalArgumentException("Settings cannot be null"); } @@ -1128,8 +1127,10 @@ public void setAllFromJson(JsonObject settings) { // Convert JSON to Setting objects Set newSettings = convertJsonToSettings(settings); - // Perform atomic update (replace all settings) - replaceAllSettings(newSettings); + // Execute the update (in one atomic operation using a transaction) + Map operationalDetails = replaceAllSettings(newSettings); + + return Op.convertToJson(operationalDetails); } /** From 614fc7106f86eca1688bde93fd47750b63db26f5 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 18 Jul 2025 17:38:52 +0200 Subject: [PATCH 240/634] fix: prevent accidental removal of all settings in `setAllFromJson` by validating non-empty input #11639 --- .../iq/dataverse/settings/SettingsServiceBean.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index cd497d91329..5ff0effedff 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -1127,10 +1127,16 @@ public JsonObjectBuilder setAllFromJson(JsonObject settings) { // Convert JSON to Setting objects Set newSettings = convertJsonToSettings(settings); - // Execute the update (in one atomic operation using a transaction) - Map operationalDetails = replaceAllSettings(newSettings); + // Perform atomic update (replace all settings) + // We don't allow to completely wipe all settings coming from JSON here, so no acciddents happen. + // (It's completely unrealistic someone would try to remove all settings and leave it at that.) + if (newSettings != null && !newSettings.isEmpty()) { + // Execute the update (in one atomic operation using a transaction) + Map operationalDetails = replaceAllSettings(newSettings); - return Op.convertToJson(operationalDetails); + return Op.convertToJson(operationalDetails); + } + throw new IllegalArgumentException("Settings cannot be empty - you'd wipe the entire configuration."); } /** From 580039b39ac083763a0f318eabbb0adb3c052199 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 21 Jul 2025 11:27:49 +0200 Subject: [PATCH 241/634] fix: ensure transactional integrity in `replaceAllSettings` using self-invocation via EJB reference #11639 Added an `EJB` self-reference in `SettingsServiceBean` to ensure proper handling of transactions, with updates to `replaceAllSettings` to use `@Transactional(REQUIRES_NEW)`. Updated method calls and comments to reflect the required use of self-invocation for EJB functionalities. --- .../dataverse/settings/SettingsServiceBean.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 5ff0effedff..05fe56875b3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -750,6 +750,15 @@ public static SettingsServiceBean.Key parse(String key) { @PersistenceContext EntityManager em; + /** + * A reference to the current instance of the SettingsServiceBean. + * Used when self-invocation is required for internal method calls + * within the same bean to ensure that all EJB functionalities + * such as transactions and security are properly applied. + */ + @EJB + private SettingsServiceBean self; + @EJB ActionLogServiceBean actionLogSvc; @@ -1132,7 +1141,8 @@ public JsonObjectBuilder setAllFromJson(JsonObject settings) { // (It's completely unrealistic someone would try to remove all settings and leave it at that.) if (newSettings != null && !newSettings.isEmpty()) { // Execute the update (in one atomic operation using a transaction) - Map operationalDetails = replaceAllSettings(newSettings); + // Note: We need to call via self-reference so the EJB container can create a transaction as intended. + Map operationalDetails = self.replaceAllSettings(newSettings); return Op.convertToJson(operationalDetails); } @@ -1213,6 +1223,9 @@ static JsonObjectBuilder convertToJson(Map operationalDetails) { * - Updates the content of existing settings that match the keys in the provided new settings. * - Creates new settings that are not present in the database. * + * If calling this method from within this class, make sure to use an EJB injected self-reference to it. + * Otherwise, the EJB container will not be able to provide a transaction as intended by {@code @Transactional}. + * * @param newSettings the set of new settings to replace the existing ones. * Each setting is uniquely identified by its name and language. * Must not be null (it may be empty). @@ -1220,7 +1233,7 @@ static JsonObjectBuilder convertToJson(Map operationalDetails) { * are the settings involved, and the values are the types of operations * performed (CREATED, UPDATED, DELETED). */ - @Transactional + @Transactional(Transactional.TxType.REQUIRES_NEW) public Map replaceAllSettings(Set newSettings) { Objects.requireNonNull(newSettings, "The list of new settings cannot be null (it may be empty)."); From bd85f7201bfaf26ffb73aee7a69567c5f84750d0 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 21 Jul 2025 12:26:55 +0200 Subject: [PATCH 242/634] fix,build: ensure proper late variable binding for JaCoCo configuration in `pom.xml` #11639 Updated JaCoCo settings in `pom.xml` to use `@{}` syntax for late variable binding in `argLine` for Maven Surefire and Maven Failsafe, ensuring compatibility with `prepare-agent` steps. This seems to be compatible with IntelliJ's parser for argLine - tests execute successfully. --- pom.xml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 822eaaeb79d..7a3b16a376b 100644 --- a/pom.xml +++ b/pom.xml @@ -21,8 +21,10 @@ false false integration - - + + + -Ddummy.jacoco.property=true + -Ddummy.jacoco.property=true @@ -1038,7 +1040,13 @@ ${testsToExclude} ${skipUnitTests} - ${surefire.jacoco.args} ${argLine} + + @{surefire.jacoco.args} ${argLine} **/builtin-users-spi/** @@ -1050,7 +1058,13 @@ maven-failsafe-plugin ${it.groups} - ${failsafe.jacoco.args} ${argLine} + + @{failsafe.jacoco.args} ${argLine} ${skipIntegrationTests} From c91b4fc40ee70130ebfd4d18bc7dc74a6e6328c4 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 21 Jul 2025 12:27:09 +0200 Subject: [PATCH 243/634] style: correct typo in `internalError` key in `JsonResponseBuilder` --- .../edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java b/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java index a80d54508fd..9095a40c608 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/util/JsonResponseBuilder.java @@ -131,7 +131,7 @@ public JsonResponseBuilder requestContentType(HttpServletRequest request) { * @return The enhanced builder */ public JsonResponseBuilder internalError(Throwable ex) { - this.entityBuilder.add("interalError", ex.getClass().getSimpleName()); + this.entityBuilder.add("internalError", ex.getClass().getSimpleName()); if (ex.getCause() != null) { this.entityBuilder.add("internalCause", ex.getCause().getClass().getSimpleName()); } From 4b5b6896a124c549c3fef7fd8b3e4f2a56da6e08 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 22 Jul 2025 18:33:08 +0200 Subject: [PATCH 244/634] fix,style: update SQL and class definitions to standardize lowercase table names and constraint naming conventions #11639 Without converting to this PostgreSQL convention, the checks to avoid adding existing constraints failed, resulting in failing Flyway migrations during startup. --- .../iq/dataverse/settings/Setting.java | 2 +- src/main/resources/db/migration/V6.7.0.1.sql | 20 +++++++++--------- src/main/resources/db/migration/V6.7.0.2.sql | 21 +++++++++---------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java b/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java index 0f240302366..e187d3db1cc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/Setting.java @@ -32,7 +32,7 @@ }) @Entity @Table(uniqueConstraints = { - @UniqueConstraint(name = "UC_setting_name_lang", columnNames = {"name", "lang"}), + @UniqueConstraint(name = "uc_setting_name_lang", columnNames = {"name", "lang"}), }) public class Setting implements Serializable { diff --git a/src/main/resources/db/migration/V6.7.0.1.sql b/src/main/resources/db/migration/V6.7.0.1.sql index cfe1e75d0b5..656ef04d572 100644 --- a/src/main/resources/db/migration/V6.7.0.1.sql +++ b/src/main/resources/db/migration/V6.7.0.1.sql @@ -6,7 +6,7 @@ DO $$ base_setting_content TEXT; format_settings_cursor CURSOR FOR SELECT name, content - FROM Setting + FROM setting WHERE name LIKE ':TabularIngestSizeLimit:%' AND lang IS NULL; format_record RECORD; @@ -18,7 +18,7 @@ DO $$ BEGIN -- Check if there are any format-specific settings SELECT EXISTS( - SELECT 1 FROM Setting + SELECT 1 FROM setting WHERE name LIKE ':TabularIngestSizeLimit:%' AND lang IS NULL ) INTO has_format_settings; @@ -31,7 +31,7 @@ DO $$ -- Get the base setting (without format suffix) if it exists SELECT content INTO base_setting_content - FROM Setting + FROM setting WHERE name = ':TabularIngestSizeLimit' AND lang IS NULL; @@ -42,7 +42,7 @@ DO $$ format_value := base_setting_content::BIGINT; json_object := json_object || jsonb_build_object('default', format_value); EXCEPTION WHEN invalid_text_representation THEN - RAISE WARNING 'Base TabularIngestSizeLimit setting contains non-numeric value: %. Setting to 0 (disabling ingest!).', base_setting_content; + RAISE WARNING 'Base TabularIngestSizeLimit setting contains non-numeric value: %. Setting it to 0 (disabling ingest!).', base_setting_content; json_object := json_object || jsonb_build_object('default', 0); END; END IF; @@ -57,7 +57,7 @@ DO $$ format_value := format_record.content::BIGINT; json_object := json_object || jsonb_build_object(format_name, format_value); EXCEPTION WHEN invalid_text_representation THEN - warning_message := format('Format-specific TabularIngestSizeLimit setting %s contains non-numeric value: %s. Setting to 0 (disabling ingest!).', + warning_message := format('Format-specific TabularIngestSizeLimit setting %s contains non-numeric value: %s. Setting it to 0 (disabling ingest!).', format_record.name, format_record.content); RAISE WARNING '%', warning_message; json_object := json_object || jsonb_build_object(format_name, 0); @@ -65,13 +65,13 @@ DO $$ END LOOP; -- Insert or update the new JSON-based setting - INSERT INTO Setting (name, content, lang) + INSERT INTO setting (name, content, lang) VALUES (':TabularIngestSizeLimit', json_object::TEXT, NULL) ON CONFLICT (name) WHERE lang IS NULL DO UPDATE SET content = EXCLUDED.content; -- Delete all format-specific settings - DELETE FROM Setting + DELETE FROM setting WHERE name LIKE ':TabularIngestSizeLimit:%' AND lang IS NULL; @@ -81,8 +81,8 @@ DO $$ -- 2. Migrate BuiltinUsers.KEY to the new setting name DO $$ BEGIN - IF EXISTS (SELECT 1 FROM Setting WHERE name = 'BuiltinUsers.KEY') THEN - INSERT INTO Setting (name, lang, content) VALUES (':BuiltinUsersKey', NULL, (SELECT content FROM Setting WHERE name = 'BuiltinUsers.KEY')); - DELETE FROM Setting WHERE name = 'BuiltinUsers.KEY'; + IF EXISTS (SELECT 1 FROM setting WHERE name = 'BuiltinUsers.KEY') THEN + INSERT INTO setting (name, lang, content) VALUES (':BuiltinUsersKey', NULL, (SELECT content FROM setting WHERE name = 'BuiltinUsers.KEY')); + DELETE FROM setting WHERE name = 'BuiltinUsers.KEY'; END IF; END $$; \ No newline at end of file diff --git a/src/main/resources/db/migration/V6.7.0.2.sql b/src/main/resources/db/migration/V6.7.0.2.sql index 177f88332b7..cd4b5018b1d 100644 --- a/src/main/resources/db/migration/V6.7.0.2.sql +++ b/src/main/resources/db/migration/V6.7.0.2.sql @@ -4,32 +4,31 @@ -- 3. Add NOT NULL constraints and unique constraint for name+lang pairs -- First, update any existing NULL lang values to empty string -UPDATE Setting SET lang = '' WHERE lang IS NULL; +UPDATE setting SET lang = '' WHERE lang IS NULL; -- Postgres doesn't support IF NOT EXISTS for ALTER COLUMN or ADD CONSTRAINT, so we need conditional logic DO $$ BEGIN -- Only alter columns if they need to be changed IF EXISTS (SELECT 1 FROM information_schema.columns - WHERE table_name = 'Setting' AND column_name = 'name' + WHERE table_name = 'setting' AND column_name = 'name' AND (data_type = 'text' OR is_nullable = 'YES')) THEN ALTER TABLE setting ALTER COLUMN name TYPE VARCHAR(255); - ALTER TABLE Setting ALTER COLUMN name SET NOT NULL; + ALTER TABLE setting ALTER COLUMN name SET NOT NULL; END IF; IF EXISTS (SELECT 1 FROM information_schema.columns - WHERE table_name = 'Setting' AND column_name = 'lang' + WHERE table_name = 'setting' AND column_name = 'lang' AND (data_type = 'text' OR is_nullable = 'YES')) THEN - ALTER TABLE Setting ALTER COLUMN lang TYPE VARCHAR(10); - ALTER TABLE Setting ALTER COLUMN lang SET NOT NULL; - ALTER TABLE Setting ALTER COLUMN lang SET DEFAULT ''; + ALTER TABLE setting ALTER COLUMN lang TYPE VARCHAR(10); + ALTER TABLE setting ALTER COLUMN lang SET NOT NULL; + ALTER TABLE setting ALTER COLUMN lang SET DEFAULT ''; END IF; IF NOT EXISTS (SELECT 1 FROM information_schema.table_constraints - WHERE table_name = 'Setting' - AND constraint_name = 'UC_setting_name_lang' + WHERE table_name = 'setting' + AND constraint_name = 'uc_setting_name_lang' AND constraint_type = 'UNIQUE') THEN - ALTER TABLE Setting ADD CONSTRAINT UC_setting_name_lang UNIQUE (name, lang); + ALTER TABLE setting ADD CONSTRAINT uc_setting_name_lang UNIQUE (name, lang); END IF; - END $$; From c7434a5cae652e811cfa97cb8cdbf1cddd51fd3e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 22 Jul 2025 18:33:33 +0200 Subject: [PATCH 245/634] fix: update Flyway migration to replace constraints and indexes in `setting` table for consistent validation #11639 Added conditional logic to drop outdated constraints and the unique index in the `setting` table. New validation imposes restrictions with an empty `lang` as the default. Revamped design aligns with API validation requirements. --- src/main/resources/db/migration/V6.7.0.2.sql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/resources/db/migration/V6.7.0.2.sql b/src/main/resources/db/migration/V6.7.0.2.sql index cd4b5018b1d..07a568f0834 100644 --- a/src/main/resources/db/migration/V6.7.0.2.sql +++ b/src/main/resources/db/migration/V6.7.0.2.sql @@ -9,6 +9,13 @@ UPDATE setting SET lang = '' WHERE lang IS NULL; -- Postgres doesn't support IF NOT EXISTS for ALTER COLUMN or ADD CONSTRAINT, so we need conditional logic DO $$ BEGIN + -- These database constraints were added with Dataverse 4.15, but they had no representation in the code, + -- not even a comment about their existence. See also Flyway script V4.16.0.1__5303-addColumn-to-settingTable.sql. + -- We are going to replace them with the new design here, using an empty lang as default. + -- Before, lang could be more or less anything. Now we do imply restrictions on validation within the API. + ALTER TABLE setting DROP CONSTRAINT IF EXISTS non_empty_lang; + DROP INDEX IF EXISTS unique_settings; + -- Only alter columns if they need to be changed IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'setting' AND column_name = 'name' From ff45c8096afc63a3fa5582f37caf3599bec4b5b8 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 22 Jul 2025 18:34:28 +0200 Subject: [PATCH 246/634] fix: update `get` method in `Admin` API to handle third parameter for settings retrieval #11639 This was wrongly requesting the setting, using the language parameter as the default value if not configured. --- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 1dc6e70fa3f..d81b5115ea1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -281,7 +281,7 @@ public Response getSetting(@PathParam("name") String name, @PathParam("lang") St SettingsServiceBean.validateSettingName(name); SettingsServiceBean.validateSettingLang(lang); - String content = settingsSvc.get(name, lang); + String content = settingsSvc.get(name, lang, null); return (content != null) ? ok(content) : notFound("Setting " + name + " for language " + lang + " not found."); } catch (IllegalArgumentException iae) { return error(Response.Status.BAD_REQUEST, iae.getMessage()); From 11890681720d13ca349f3dcab1b8489ea4dd4397 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 22 Jul 2025 18:42:57 +0200 Subject: [PATCH 247/634] test: add nested test cases for settings API in `AdminIT` #11639 - Introduced new nested test class `SettingsAPI` to `AdminIT` with focused test cases for settings retrieval, updates, and localization handling. - Added utility methods in `UtilIT` to support language-specific settings operations. - Ensured thorough cleanup in tests to prevent state leakage. --- .../edu/harvard/iq/dataverse/api/AdminIT.java | 187 ++++++++++++++++-- .../edu/harvard/iq/dataverse/api/UtilIT.java | 5 + 2 files changed, 175 insertions(+), 17 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java index b48c5507a54..8143796cbf1 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java @@ -1,38 +1,48 @@ package edu.harvard.iq.dataverse.api; -import io.restassured.RestAssured; -import io.restassured.path.json.JsonPath; -import io.restassured.response.Response; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinAuthenticationProvider; import edu.harvard.iq.dataverse.authorization.providers.oauth2.impl.GitHubOAuth2AP; import edu.harvard.iq.dataverse.authorization.providers.oauth2.impl.OrcidOAuth2AP; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - +import io.restassured.RestAssured; +import io.restassured.path.json.JsonPath; +import io.restassured.response.Response; import jakarta.json.Json; import jakarta.json.JsonArray; +import jakarta.json.JsonObject; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; - - +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.UUID; import java.util.logging.Logger; -import static jakarta.ws.rs.core.Response.Status.*; -import static org.hamcrest.CoreMatchers.*; +import static io.restassured.RestAssured.given; +import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; +import static jakarta.ws.rs.core.Response.Status.CREATED; +import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; +import static jakarta.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR; +import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; +import static jakarta.ws.rs.core.Response.Status.OK; +import static jakarta.ws.rs.core.Response.Status.UNAUTHORIZED; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; public class AdminIT { @@ -45,7 +55,150 @@ public class AdminIT { public static void setUp() { RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); } - + + @Nested + class SettingsAPI { + + static final SettingsServiceBean.Key harmlessSetting = SettingsServiceBean.Key.InstallationName; + static final String harmlessValue = "Test Instance Name"; + static final String language = "fr"; + static final String harmlessL10nValue = "Nom de l'instance de test"; + + @AfterAll + static void destroy() { + // No leftover settings after breaking tests! + UtilIT.deleteSetting(harmlessSetting); + UtilIT.deleteSetting(harmlessSetting, language); + } + + @Test + void testSettingsRoundTrip() { + Assumptions.assumeTrue(UtilIT.getSetting(harmlessSetting).statusCode() == NOT_FOUND.getStatusCode(), "Harmless setting should not exist yet."); + Assumptions.assumeTrue(UtilIT.getSetting(harmlessSetting, language).statusCode() == NOT_FOUND.getStatusCode(), "Harmless localized setting should not exist yet."); + + // Step 0: Add a localized setting so we can make sure the put all can cope with that, too. + UtilIT.setSetting(harmlessSetting, harmlessL10nValue, language); + + // Step 1: Get current settings state + Response getResponse = given() + .when() + .get("/api/admin/settings"); + + getResponse.then() + .assertThat() + .statusCode(OK.getStatusCode()) + .contentType("application/json") + .body("status", equalTo("OK")) + .body("data.'"+harmlessSetting+"/lang/"+language+"'", equalTo(harmlessL10nValue)); + + // Store original settings as JsonObject for later restoration + JsonObject originalSettings = Json.createReader(getResponse.body().asInputStream()) + .readObject() + .getJsonObject("data"); + + // Step 2: Set our harmless test setting using UtilIT + Response setResponse = UtilIT.setSetting(harmlessSetting.toString(), harmlessValue); + setResponse.then() + .assertThat() + .statusCode(OK.getStatusCode()); + + // Step 3: Verify the harmless setting was set + Response verifySetResponse = UtilIT.getSetting(harmlessSetting); + + verifySetResponse.then() + .assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", equalTo(harmlessValue)); + + // Step 4: Put back the original settings (this is what we're testing) + Response putResponse = given() + //.header("X-Dataverse-key", "") + .header("Content-Type", "application/json") + .body(originalSettings.toString()) + .when() + .put("/api/admin/settings"); + + putResponse.then() + .assertThat() + .statusCode(OK.getStatusCode()) + .body("status", equalTo("OK")) + .body("message.message", containsString("successfully updated")); + + // Step 5: Verify the harmless setting is gone (restored to original state) + Response verifyRestoredResponse = given() + //.header("X-Dataverse-key", "") + .when() + .get("/api/admin/settings" + harmlessSetting.toString()); + + verifyRestoredResponse.then() + .assertThat() + .statusCode(NOT_FOUND.getStatusCode()); // Should not exist anymore + + // Step 6: Verify overall settings state matches original + Response finalGetResponse = given() + //.header("X-Dataverse-key", "") + .when() + .get("/api/admin/settings"); + + finalGetResponse.then() + .assertThat() + .statusCode(OK.getStatusCode()); + + // Store original settings as JsonObject for later restoration + JsonObject finalSettings = Json.createReader(getResponse.body().asInputStream()) + .readObject() + .getJsonObject("data"); + + // Verify the settings are back to original state (our test setting should be absent) + assertFalse(finalSettings.containsKey(harmlessSetting.toString()), "Harmless setting should not exist in restored settings"); + + // Cleanup: delete the localized setting + UtilIT.deleteSetting(harmlessSetting, language); + } + + @Test + void testGetAllSettingsWithLocalization() { + int statusCode = UtilIT.getSetting(harmlessSetting, language).statusCode(); + Assumptions.assumeTrue(statusCode == NOT_FOUND.getStatusCode(), "Harmless localized setting should not exist yet. Status Code: " + statusCode); + + // Given + UtilIT.setSetting(harmlessSetting, harmlessL10nValue, language); + + // When + Response getResponse = given() + .when() + .get("/api/admin/settings"); + + // Then + getResponse.then() + .assertThat() + .statusCode(OK.getStatusCode()) + .contentType("application/json") + .body("status", equalTo("OK")) + .body("data.'"+harmlessSetting+"/lang/"+language+"'", equalTo(harmlessL10nValue)); + + // Cleanup + UtilIT.deleteSetting(harmlessSetting, language); + } + + @Test + void testPutAllSettingsWithEmptyJson() { + // Test error handling for empty JSON + Response response = given() + //.header("X-Dataverse-key", UtilIT.getSuperuserApiToken()) + .header("Content-Type", "application/json") + .body("{}") + .when() + .put("/api/admin/settings"); + + response.then() + .assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", containsString("Empty or invalid JSON object")); + } + } + + @Test public void testListAuthenticatedUsers() throws Exception { Response anon = UtilIT.listAuthenticatedUsers(testNonSuperuserApiToken); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index eba8181e566..aa11ceb0d18 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2511,6 +2511,11 @@ static Response getSetting(SettingsServiceBean.Key settingKey) { Response response = given().when().get("/api/admin/settings/" + settingKey); return response; } + + static Response getSetting(SettingsServiceBean.Key settingKey, String language) { + Response response = given().when().get("/api/admin/settings/" + settingKey + "/lang/" + language); + return response; + } static Response setSetting(SettingsServiceBean.Key settingKey, String value) { Response response = given().body(value).when().put("/api/admin/settings/" + settingKey); From 982e0921524d8d15e9a6101c00e91d14a6f2b759 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 23 Jul 2025 17:18:11 +0200 Subject: [PATCH 248/634] docs: add release note and API docs for #11639 --- .../11639-db-opts-idempotency.md | 21 ++++++++++ doc/sphinx-guides/source/api/native-api.rst | 38 ++++++++++++++++--- .../source/installation/config.rst | 2 + 3 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 doc/release-notes/11639-db-opts-idempotency.md diff --git a/doc/release-notes/11639-db-opts-idempotency.md b/doc/release-notes/11639-db-opts-idempotency.md new file mode 100644 index 00000000000..2af82f4ffda --- /dev/null +++ b/doc/release-notes/11639-db-opts-idempotency.md @@ -0,0 +1,21 @@ +## Database Settings Cleanup + +With this release, we remove some legacy specialties around Database Settings and provide better Admin API endpoints for them. + +Most important changes: + +1. Setting `BuiltinUsers.KEY` was renamed to `:BuiltinUsersKey`, aligned with our general naming pattern for options. +2. Setting `:TabularIngestSizeLimit` no longer uses suffixes for formats and becomes a JSON-based setting instead. +3. If set, both settings will be migrated to their new form automatically for you (Flyway migration). +4. You can no longer (accidentally) create or use arbitrary setting names or languages. + All Admin API endpoints for settings now validate setting names and languages for existence and compliance. + +As an administrator of a Dataverse instance, you can now make use of enhanced Bulk Operations on the Settings Admin API: + +1. Retrieving all settings as JSON via `GET /api/admin/settings` supports localized options now, too. +2. You can replace all existing settings in an idempotent way sending JSON to `PUT /api/admin/settings`. + This will create, update and remove settings as necessary in one atomic operation. + The new endpoint is especially useful to admins using GitOps or other automations. + It allows control over all Database Settings from a single source without risking an undefined state. + +Note: Despite the validation of setting names and languages, the content of any database setting is still not being validated when using the Settings Admin API! diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 7dc50e7a532..c55fc424496 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -6775,7 +6775,11 @@ If the PID is not managed by Dataverse, this call will report if the PID is reco Admin ----- -This is the administrative part of the API. For security reasons, it is absolutely essential that you block it before allowing public access to a Dataverse installation. Blocking can be done using settings. See the ``post-install-api-block.sh`` script in the ``scripts/api`` folder for details. See :ref:`blocking-api-endpoints` in Securing Your Installation section of the Configuration page of the Installation Guide. +This is the administrative part of the API. +For security reasons, it is absolutely essential that you block it before allowing public access to a Dataverse installation. +Blocking can be done using settings. +See the ``post-install-api-block.sh`` script in the ``scripts/api`` folder for details. +See :ref:`blocking-api-endpoints` in Securing Your Installation section of the Configuration page of the Installation Guide. List All Database Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -6784,13 +6788,29 @@ List all settings:: GET http://$SERVER/api/admin/settings -Configure Database Setting -~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. _settings_put_bulk: + +Configure All Database Settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Replace all settings in a single idempotent and atomic operation:: + + PUT http://$SERVER/api/admin/settings + +See JSON ``data`` object in output of ``GET /api/admin/settings`` for the JSON input structure for this endpoint. +The :doc:`../installation/config` page of the Installation Guide has a :ref:`complete list of all the available settings `. + +Configure Single Database Setting +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Sets setting ``name`` to the body of the request:: PUT http://$SERVER/api/admin/settings/$name +Sets a localized setting ``name`` for locale/language ``lang`` to the body of the request:: + + PUT http://$SERVER/api/admin/settings/$name/lang/$lang + Get Single Database Setting ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -6798,13 +6818,21 @@ Get the setting under ``name``:: GET http://$SERVER/api/admin/settings/$name -Delete Database Setting -~~~~~~~~~~~~~~~~~~~~~~~ +Gets a localized setting under ``name`` for locale/language ``lang``:: + + GET http://$SERVER/api/admin/settings/$name/lang/$lang + +Delete Single Database Setting +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Delete the setting under ``name``:: DELETE http://$SERVER/api/admin/settings/$name +Delete a localized setting under ``name`` for locale/language ``lang``:: + + DELETE http://$SERVER/api/admin/settings/$name/lang/$lang + .. _list-all-feature-flags: List All Feature Flags diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 8722da19b78..f064cc51a06 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3879,6 +3879,8 @@ The most commonly used configuration options are listed first. The pattern you will observe in curl examples below is that an HTTP ``PUT`` is used to add or modify a setting. If you perform an HTTP ``GET`` (the default when using curl), the output will contain the value of the setting, if it has been set. You can also do a ``GET`` of all settings with ``curl http://localhost:8080/api/admin/settings`` which you may want to pretty-print by piping the output through a tool such as jq by appending ``| jq .``. If you want to remove a setting, use an HTTP ``DELETE`` such as ``curl -X DELETE http://localhost:8080/api/admin/settings/:GuidesBaseUrl`` . +For your convenience, there is also an Admin API endpoint to :ref:`bulk manage database settings in an atomic, idempotent fashion `. + .. _:BlockedApiPolicy: :BlockedApiPolicy (Deprecated) From 1b06dfb868ca84443d3a541b853d478a854cfc1c Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 23 Jul 2025 18:27:49 +0200 Subject: [PATCH 249/634] feat: add `UNCHANGED` state handling in `SettingsServiceBean` operations #11639 - Introduced `UNCHANGED` as a new operation type for settings. - Updated `convertToJson` method to include unchanged settings in JSON output. - Enhanced `replaceAllSettings` logic to track unchanged settings when content remains identical. - The reason for this is idempotency: if nothing changed, nothing should be done. Replacing the present content of an existing setting is unnecessary, wasting CPU cycles, and we also want the admin to be informed about what actually is getting changed. Before, all of these unchanged elements would be counted as "updated". --- .../settings/SettingsServiceBean.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 05fe56875b3..5f82030eb75 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -1191,7 +1191,8 @@ static Set convertJsonToSettings(JsonObject settings) { static enum Op { UPDATED, CREATED, - DELETED; + DELETED, + UNCHANGED; static JsonObjectBuilder convertToJson(Map operationalDetails) { // Create a nice represenation of what happened as Json @@ -1199,6 +1200,7 @@ static JsonObjectBuilder convertToJson(Map operationalDetails) { JsonArrayBuilder created = Json.createArrayBuilder(); JsonArrayBuilder updated = Json.createArrayBuilder(); JsonArrayBuilder deleted = Json.createArrayBuilder(); + JsonArrayBuilder unchanged = Json.createArrayBuilder(); operationalDetails.forEach((setting, op) -> { String name = convertToJsonKey(setting); @@ -1206,13 +1208,15 @@ static JsonObjectBuilder convertToJson(Map operationalDetails) { case CREATED -> created.add(name); case UPDATED -> updated.add(name); case DELETED -> deleted.add(name); + case UNCHANGED -> unchanged.add(name); } }); return jbo .add("created", created) .add("updated", updated) - .add("deleted", deleted); + .add("deleted", deleted) + .add("unchanged", unchanged); } } @@ -1268,10 +1272,14 @@ public Map replaceAllSettings(Set newSettings) { // Setting exists in both - update with new values } else { Setting newSetting = newByKey.get(key); - // We use the already managed entity and update it with the content of the new setting. - // (This means we don't need to call em.merge(), the ORM will track and execute it for us.) - existingSetting.setContent(newSetting.getContent()); - opsTracking.put(existingSetting, Op.UPDATED); + if (existingSetting.getContent().equals(newSetting.getContent())) { + opsTracking.put(existingSetting, Op.UNCHANGED); + } else { + // We use the already managed entity and update it with the content of the new setting. + // (This means we don't need to call em.merge(), the ORM will track and execute it for us.) + existingSetting.setContent(newSetting.getContent()); + opsTracking.put(existingSetting, Op.UPDATED); + } } } From d500650ceddcd816c3b898dea8311358d9d3941a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 23 Jul 2025 18:28:05 +0200 Subject: [PATCH 250/634] test: add unit tests for `replaceAllSettings` in `SettingsServiceBean` #11639 - Introduced a nested test class to validate `replaceAllSettings` behavior. - Added test cases to cover scenarios for null input, updates, deletions, creations, and unchanged settings. - Ensured proper mock verifications to validate interactions with `EntityManager` and prevent state leakage in tests. --- .../settings/SettingsServiceBeanTest.java | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java index bcc82933226..2b0ba2871f9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java @@ -4,6 +4,7 @@ import jakarta.json.JsonObject; import jakarta.persistence.EntityManager; import jakarta.persistence.TypedQuery; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -16,12 +17,17 @@ import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; class SettingsServiceBeanTest { @@ -315,4 +321,85 @@ void testConvertJsonToSettings_complexJsonValue() { } + + @Nested + class ReplaceAllSettingsTest { + + static TypedQuery typedQuery = mock(TypedQuery.class); + static EntityManager em = mock(EntityManager.class); + static SettingsServiceBean settingsServiceBean = new SettingsServiceBean(); + + @BeforeAll + static void setup() { + settingsServiceBean.em = em; + + when(em.createNamedQuery( + ArgumentMatchers.eq("Setting.findAll"), + ArgumentMatchers.eq(Setting.class))) + .thenReturn(typedQuery); + } + + @AfterEach + void reset() { + // After each test, we need to clear the invocations for test isolation. + clearInvocations(em); + } + + @Test + void testReplaceAllSettings_null() { + // When/Then + NullPointerException exception = assertThrows(NullPointerException.class, + () -> settingsServiceBean.replaceAllSettings(null)); + assertEquals("The list of new settings cannot be null (it may be empty).", exception.getMessage()); + } + + @Test + void testReplaceAllSettings_updateDeleteCreate() { + // Given + Setting existingSetting1 = new Setting(":Key1", "Value1"); + Setting existingSetting2 = new Setting(":Key2", "Value2"); + Setting newSetting1 = new Setting(":Key1", "UpdatedValue1"); + Setting newSetting3 = new Setting(":Key3", "Value3"); + + when(typedQuery.getResultList()).thenReturn(List.of(existingSetting1, existingSetting2)); + + // When + Map result = settingsServiceBean.replaceAllSettings(Set.of(newSetting1, newSetting3)); + + // Then + assertEquals(3, result.size()); + assertEquals(SettingsServiceBean.Op.UPDATED, result.get(existingSetting1)); + assertEquals(SettingsServiceBean.Op.DELETED, result.get(existingSetting2)); + assertEquals(SettingsServiceBean.Op.CREATED, result.get(newSetting3)); + // We cannot track the em.merge() call in this unit-test, as this happens in ORM code, beyond our reach. + // Thus check the update to the ORM-tracked entity happened. + assertEquals("UpdatedValue1", existingSetting1.getContent()); + + // Verify interactions + verify(em).remove(existingSetting2); + verify(em).persist(newSetting3); + verify(em).flush(); // verify persistence is enforced + } + + @Test + void testReplaceAllSettings_noChanges() { + // Given + Setting existingSetting = new Setting(":Key1", "Value1"); + Setting newSetting = new Setting(":Key1", "Value1"); + + when(typedQuery.getResultList()).thenReturn(List.of(existingSetting)); + + // When + Map result = settingsServiceBean.replaceAllSettings(Set.of(newSetting)); + + // Then + assertEquals(1, result.size()); + assertEquals(SettingsServiceBean.Op.UNCHANGED, result.get(existingSetting)); + + // Verify no interactions causing change + verify(em, never()).persist(any(Setting.class)); + verify(em, never()).remove(any(Setting.class)); + verify(em, never()).merge(any(Setting.class)); + } + } } \ No newline at end of file From 4758f86d957f5f3459ef7457157ebed6fa175ae0 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 24 Jul 2025 10:50:44 +0200 Subject: [PATCH 251/634] refactor: introduce `SettingsValidationException` for enhanced error handling in settings logic #11639 - Replaced `IllegalArgumentException` with a custom `SettingsValidationException` for improved specificity. - Updated all related methods in `SettingsServiceBean` and `Admin` API to use the new exception. - Added `SettingsValidationException` class as an application-level exception with rollback support. - This way we don't see wrapped exceptions (either as EJBException or EJBTransactionRollbackException), as we tell the EJB system this is a known exception. - It may also make Copilot Review happy, as we are limiting the chances of leaking internal information with this specific exception. --- .../edu/harvard/iq/dataverse/api/Admin.java | 26 +++++++++---------- .../settings/SettingsServiceBean.java | 26 +++++++++---------- .../settings/SettingsValidationException.java | 10 +++++++ .../settings/SettingsServiceBeanTest.java | 4 +-- 4 files changed, 38 insertions(+), 28 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/settings/SettingsValidationException.java diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index d81b5115ea1..de7890250e9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -18,6 +18,7 @@ import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.settings.SettingsValidationException; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.cache.CacheFactoryBean; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; @@ -228,9 +229,8 @@ public Response putAllSettings(JsonObject settings) { // Transfer to domain objects and deeper validation to be handled by the service layer. JsonObjectBuilder successfullOperations = settingsSvc.setAllFromJson(settings); return ok("All database options successfully updated.", successfullOperations); - - } catch (IllegalArgumentException iae) { - return error(Response.Status.BAD_REQUEST, iae.getMessage()); + } catch (SettingsValidationException sve) { + return error(Response.Status.BAD_REQUEST, sve.getMessage()); } } @@ -242,8 +242,8 @@ public Response putSetting(@PathParam("name") String name, String content) { Setting s = settingsSvc.set(name, content); return ok("Setting " + name + " added."); - } catch (IllegalArgumentException iae) { - return error(Response.Status.BAD_REQUEST, iae.getMessage()); + } catch (SettingsValidationException sve) { + return error(Response.Status.BAD_REQUEST, sve.getMessage()); } } @@ -256,8 +256,8 @@ public Response putSettingLang(@PathParam("name") String name, @PathParam("lang" Setting s = settingsSvc.set(name, lang, content); return ok("Setting " + name + " added for language " + lang + "."); - } catch (IllegalArgumentException iae) { - return error(Response.Status.BAD_REQUEST, iae.getMessage()); + } catch (SettingsValidationException sve) { + return error(Response.Status.BAD_REQUEST, sve.getMessage()); } } @@ -283,8 +283,8 @@ public Response getSetting(@PathParam("name") String name, @PathParam("lang") St String content = settingsSvc.get(name, lang, null); return (content != null) ? ok(content) : notFound("Setting " + name + " for language " + lang + " not found."); - } catch (IllegalArgumentException iae) { - return error(Response.Status.BAD_REQUEST, iae.getMessage()); + } catch (SettingsValidationException sve) { + return error(Response.Status.BAD_REQUEST, sve.getMessage()); } } @@ -296,8 +296,8 @@ public Response deleteSetting(@PathParam("name") String name) { settingsSvc.delete(name); return ok("Setting " + name + " deleted."); - } catch (IllegalArgumentException iae) { - return error(Response.Status.BAD_REQUEST, iae.getMessage()); + } catch (SettingsValidationException sve) { + return error(Response.Status.BAD_REQUEST, sve.getMessage()); } } @@ -310,8 +310,8 @@ public Response deleteSettingLang(@PathParam("name") String name, @PathParam("la settingsSvc.delete(name, lang); return ok("Setting " + name + " for language " + lang + " deleted."); - } catch (IllegalArgumentException iae) { - return error(Response.Status.BAD_REQUEST, iae.getMessage()); + } catch (SettingsValidationException sve) { + return error(Response.Status.BAD_REQUEST, sve.getMessage()); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 5f82030eb75..4006435bf09 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -1114,23 +1114,23 @@ public JsonObject listAllAsJson() { /** * Updates all current settings from the specified JSON object. Validates the input JSON, - * converts it to a set of settings, and replaces all existing settings with the new ones + * converts it to a set of settings and replaces all existing settings with the new ones * in an atomic operation. If the settings object is null, contains invalid keys, or if the new * set of settings is empty, the method throws an appropriate exception. * * @param settings the JSON object containing the new configuration settings to be applied; must not be null * @return a JsonObjectBuilder representing the operational details of the applied updates - * @throws IllegalArgumentException if the settings object is null, contains invalid keys, or results in empty settings + * @throws SettingsValidationException if the settings object is null, contains invalid keys or results in empty settings */ public JsonObjectBuilder setAllFromJson(JsonObject settings) { if (settings == null) { - throw new IllegalArgumentException("Settings cannot be null"); + throw new SettingsValidationException("Settings cannot be null"); } // Validate the input List invalidKeys = validateKeys(settings); if (!invalidKeys.isEmpty()) { - throw new IllegalArgumentException("Invalid key(s): " + String.join(", ", invalidKeys)); + throw new SettingsValidationException("Invalid key(s): " + String.join(", ", invalidKeys)); } // Convert JSON to Setting objects @@ -1146,7 +1146,7 @@ public JsonObjectBuilder setAllFromJson(JsonObject settings) { return Op.convertToJson(operationalDetails); } - throw new IllegalArgumentException("Settings cannot be empty - you'd wipe the entire configuration."); + throw new SettingsValidationException("Settings cannot be empty - you'd wipe the entire configuration."); } /** @@ -1378,7 +1378,7 @@ public static List validateKeys(JsonObject settings) { } else { validateSettingName(key); } - } catch (IllegalArgumentException iae) { + } catch (SettingsValidationException sev) { invalidKeys.add(key); } } @@ -1387,22 +1387,22 @@ public static List validateKeys(JsonObject settings) { /** * Validates the provided setting name to ensure it meets the required format. - * Throws an {@code IllegalArgumentException} if the name is invalid, including cases + * Throws an {@code SettingsValidationException} if the name is invalid, including cases * where it contains a colon-separated suffix that is no longer supported. * * @param name The name of the setting to be validated. * It must adhere to the allowable setting name format. * Names with more than one colon, which may indicate deprecated suffix formats, are not allowed. - * @throws IllegalArgumentException if the setting name is invalid. + * @throws SettingsValidationException if the setting name is invalid. */ public static void validateSettingName(String name) { if (SettingsServiceBean.Key.parse(name) == null) { // If there is more than one colon, this may be someone trying to use the old suffix settings. // Change the error message for that slightly. if (name.replace(":","").length() < name.length() - 1) { - throw new IllegalArgumentException("The name of the setting may not have a colon separated suffix since Dataverse 6.8. Please update your scripts."); + throw new SettingsValidationException("The name of the setting may not have a colon separated suffix since Dataverse 6.8. Please update your scripts."); } - throw new IllegalArgumentException("The name of the setting is invalid."); + throw new SettingsValidationException("The name of the setting is invalid."); } } @@ -1410,15 +1410,15 @@ public static void validateSettingName(String name) { * Validates the provided language code to ensure it adheres to the ISO 639-1 format. * This method checks that the language code is not null, has a length of 2 characters, * and exists within the list of valid ISO 639-1 language codes. If the validation - * fails, an {@code IllegalArgumentException} is thrown. + * fails, an {@code SettingsValidationException} is thrown. * * @param lang the language code to be validated. It must be a non-null, * 2-character string representing a valid ISO 639-1 language code. - * @throws IllegalArgumentException if the language code is invalid. + * @throws SettingsValidationException if the language code is invalid. */ public static void validateSettingLang(String lang) { if (lang == null || lang.length() != 2 || !Arrays.asList(Locale.getISOLanguages()).contains(lang)) { - throw new IllegalArgumentException("The language '" + lang + "' is not a valid ISO 639-1 language code."); + throw new SettingsValidationException("The language '" + lang + "' is not a valid ISO 639-1 language code."); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsValidationException.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsValidationException.java new file mode 100644 index 00000000000..e02e3234675 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsValidationException.java @@ -0,0 +1,10 @@ +package edu.harvard.iq.dataverse.settings; + +import jakarta.ejb.ApplicationException; + +@ApplicationException(rollback = true) +public class SettingsValidationException extends RuntimeException { + public SettingsValidationException(String message) { + super(message); + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java index 2b0ba2871f9..ed6d5417c06 100644 --- a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java @@ -83,7 +83,7 @@ void testValidateSettingName_validNames(String name) { }) @ParameterizedTest void testValidateSettingName_invalidNames(String name, String expectedMessage) { - IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + SettingsValidationException exception = assertThrows(SettingsValidationException.class, () -> SettingsServiceBean.validateSettingName(name)); assertEquals(expectedMessage, exception.getMessage()); } @@ -106,7 +106,7 @@ void testValidateSettingLang_validLanguage(String language) { }) @ParameterizedTest void testValidateSettingLang_invalidLanguage(String language, String expectedMessage) { - IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + SettingsValidationException exception = assertThrows(SettingsValidationException.class, () -> SettingsServiceBean.validateSettingLang(language)); assertEquals(expectedMessage, exception.getMessage()); } From e33b3f7f6808edabc26413341886db98a494daef Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 24 Jul 2025 10:51:03 +0200 Subject: [PATCH 252/634] test: add validation test for invalid settings in `AdminIT` #11639 - Added `testPutAllSettingsWithInvalidSetting` to verify error handling for invalid keys. - Ensures proper response code and error message for invalid settings payload. --- .../edu/harvard/iq/dataverse/api/AdminIT.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java index 8143796cbf1..83a62d41b9f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java @@ -185,7 +185,6 @@ void testGetAllSettingsWithLocalization() { void testPutAllSettingsWithEmptyJson() { // Test error handling for empty JSON Response response = given() - //.header("X-Dataverse-key", UtilIT.getSuperuserApiToken()) .header("Content-Type", "application/json") .body("{}") .when() @@ -196,6 +195,21 @@ void testPutAllSettingsWithEmptyJson() { .statusCode(BAD_REQUEST.getStatusCode()) .body("message", containsString("Empty or invalid JSON object")); } + + @Test + void testPutAllSettingsWithInvalidSetting() { + // Test error handling for empty JSON + Response response = given() + .header("Content-Type", "application/json") + .body("{\":Test1\": \"Foobar\", \":Test2\": \"Foobar\" }") + .when() + .put("/api/admin/settings"); + + response.then() + .assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", containsString("Invalid key(s): :Test1, :Test2")); + } } From 08a28615671c8a0fb4c8b16c1a994ec3efff866a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 24 Jul 2025 10:55:21 +0200 Subject: [PATCH 253/634] fix: replace `@Transactional` with `@TransactionAttribute` in `SettingsServiceBean` #11639 As this is an EJBean and not under CDI control, we must use EJB Transaction Annotations. Otherwise, the behavior is undefined, as the container may or may not look after transactions as we want it to. --- .../harvard/iq/dataverse/settings/SettingsServiceBean.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 4006435bf09..c29f94e2672 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -7,6 +7,8 @@ import edu.harvard.iq.dataverse.util.json.JsonUtil; import jakarta.ejb.EJB; import jakarta.ejb.Stateless; +import jakarta.ejb.TransactionAttribute; +import jakarta.ejb.TransactionAttributeType; import jakarta.inject.Named; import jakarta.json.Json; import jakarta.json.JsonArray; @@ -17,7 +19,6 @@ import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; -import jakarta.transaction.Transactional; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -1237,7 +1238,7 @@ static JsonObjectBuilder convertToJson(Map operationalDetails) { * are the settings involved, and the values are the types of operations * performed (CREATED, UPDATED, DELETED). */ - @Transactional(Transactional.TxType.REQUIRES_NEW) + @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) public Map replaceAllSettings(Set newSettings) { Objects.requireNonNull(newSettings, "The list of new settings cannot be null (it may be empty)."); From 5a099efc1866561c977237db7d93e13f3ec1ce8f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 24 Jul 2025 13:40:00 +0200 Subject: [PATCH 254/634] feat(ct): add database settings replacement script to configbaker image #11639 - Introduced `apply-db-settings.sh` for safely replacing database settings in a Dataverse instance. - Added logic to validate inputs, ensure dependencies (`yq`, `jq`, `wait4x`) are available, and handle API authentication via unblock keys. - Updated Dockerfile to include `yq` installation with specified version. --- modules/container-configbaker/Dockerfile | 10 +- .../scripts/apply-db-settings.sh | 129 ++++++++++++++++++ 2 files changed, 137 insertions(+), 2 deletions(-) create mode 100755 modules/container-configbaker/scripts/apply-db-settings.sh diff --git a/modules/container-configbaker/Dockerfile b/modules/container-configbaker/Dockerfile index 5532cda1a9e..9fc876a283b 100644 --- a/modules/container-configbaker/Dockerfile +++ b/modules/container-configbaker/Dockerfile @@ -23,6 +23,8 @@ ENV PATH="${PATH}:${SCRIPT_DIR}" \ ARG PKGS="bc curl dnsutils dumb-init ed jq netcat-openbsd postgresql-client" # renovate: datasource=github-releases depName=wait4x/wait4x ARG WAIT4X_VERSION="v3.2.0" +# renove: datasource=github-releases depName=mikefarah/yq +ARG YQ_VERSION="v4.47.1" # renovate: datasource=pypi depName=awscli ARG AWSCLI_VERSION="1.40.15" ARG PYTHON_PKGS="awscli==${AWSCLI_VERSION}" @@ -65,7 +67,11 @@ RUN true && \ echo "$(cat /tmp/w4x-checksum | cut -f1 -d" ") /usr/bin/wait4x.tar.gz" | sha256sum -c - && \ tar -xzf /usr/bin/wait4x.tar.gz -C /usr/bin && chmod +x /usr/bin/wait4x && \ - # 2. Python packages + # 2. yq-go \ + curl -sSfL -o /usr/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${ARCH}" && \ + chmod +x /usr/bin/yq && \ + + # 3. Python packages pipx install --global ${PYTHON_PKGS} # Get in the scripts @@ -81,7 +87,7 @@ COPY --from=solr /opt/solr/server/solr/configsets/_default ${SOLR_TEMPLATE}/ COPY maven/solr/*.xml ${SOLR_TEMPLATE}/conf/ RUN rm ${SOLR_TEMPLATE}/conf/managed-schema.xml - +WORKDIR ${SCRIPT_DIR} # Set the entrypoint to tini (as a process supervisor) ENTRYPOINT ["/usr/bin/dumb-init", "--"] # By default run a script that will print a help message and terminate diff --git a/modules/container-configbaker/scripts/apply-db-settings.sh b/modules/container-configbaker/scripts/apply-db-settings.sh new file mode 100755 index 00000000000..0d0a2ecedbc --- /dev/null +++ b/modules/container-configbaker/scripts/apply-db-settings.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash + +# [INFO]: Idempotent replacement of all database settings from a file source. + +set -euo pipefail + +function usage() { + echo "Usage: $(basename "$0") [-h] [-u instanceUrl] [-t timeout] [-c configFile] [-b unblockKey]" + echo "" + echo "Replace all Database Settings in a running Dataverse installation in an idempotent way." + echo "" + echo "Parameters:" + echo "instanceUrl - Location on container network where to reach your instance. Default: 'http://dataverse:8080'" + echo " Can be set as environment variable 'DATAVERSE_URL'." + echo " timeout - Provide how long to wait for the instance to become available (using wait4x). Default: '3m'" + echo " Can be set as environment variable 'TIMEOUT'." + echo " configFile - Path to a JSON, YAML, PROPERTIES or TOML file containing your settings. Default: '/dv/db-opts.yml'" + echo " Can be set as environment variable 'CONFIG_FILE'." + echo " unblockKey - Either string or path to a file with the Admin API Unblock Key. Optional for localhost. No default." + echo " Can be set as environment variable 'ADMIN_API_UNBLOCK_KEY'." + echo "" + echo "Note: This script will wait for the Dataverse instance to be available before executing the replacement." + echo " Be careful - this script will not stop you from deleting any vital settings." + echo "" + exit 1 +} + +### Common functions +function error { + echo "ERROR:" "$@" >&2 + exit 2 +} + +function exists { + type "$1" >/dev/null 2>&1 && return 0 + ( IFS=:; for p in $PATH; do [ -x "${p%/}/$1" ] && return 0; done; return 1 ) +} + +# Check for (the right) yq, jq, and wait4x being available +if ! exists yq; then + error "No yq executable found on PATH." +elif ! grep -q "https://github.com/mikefarah/yq" <((yq --version)); then + error "You must install yq from https://github.com/mikefarah/yq, not https://github.com/kislyuk/yq" +fi +if ! exists jq; then + error "No jq executable found on PATH." +fi +if ! exists wait4x; then + error "No wait4x executable found on PATH." +fi + +# Set some defaults as documented +DATAVERSE_URL=${DATAVERSE_URL:-"http://dataverse:8080"} +ADMIN_API_UNBLOCK_KEY=${ADMIN_API_UNBLOCK_KEY:-""} +TIMEOUT=${TIMEOUT:-"3m"} +CONFIG_FILE=${CONFIG_FILE:-"/dv/db-opts.yml"} + +while getopts "u:t:c:b:h" OPTION +do + case "$OPTION" in + u) DATAVERSE_URL="$OPTARG" ;; + t) TIMEOUT="$OPTARG" ;; + c) CONFIG_FILE="$OPTARG" ;; + b) ADMIN_API_UNBLOCK_KEY="$OPTARG" ;; + h) usage;; + \?) usage;; + esac +done +shift $((OPTIND-1)) + +# Define an auth header argument (enabling usage of different ways) +AUTH_HEADER_ARG="" + +# Check for Dataverse Unblock API Key present (option with file/env var) +# This is only required if the host is not localhost (then there may be no key necessary) +if ! [[ "${DATAVERSE_URL}" == *"://localhost"* ]] || [ -n "${ADMIN_API_UNBLOCK_KEY}" ]; then + # The argument should not be empty + if [ -z "${ADMIN_API_UNBLOCK_KEY}" ]; then + error "You must provide the Dataverse API Unblock Key to this script." + # In case it's not empty, check if it's a file path and read the key from there + elif [ -f "${ADMIN_API_UNBLOCK_KEY}" ] && [ -r "${ADMIN_API_UNBLOCK_KEY}" ]; then + echo "Reading Dataverse API Unblock Key from ${ADMIN_API_UNBLOCK_KEY}." + if ! API_KEY_FILE_CONTENT=$(cat "${ADMIN_API_UNBLOCK_KEY}" 2>/dev/null); then + error "Could not read unblock key from file ${ADMIN_API_UNBLOCK_KEY}." + fi + # Validate the key is not empty + if [ -z "${API_KEY_FILE_CONTENT}" ]; then + error "API key file ${ADMIN_API_UNBLOCK_KEY} appears empty." + fi + ADMIN_API_UNBLOCK_KEY="$API_KEY_CONTENT" + fi + # Very basic error check (as there is no clear format or formal spec for the key) + if [ ${#ADMIN_API_UNBLOCK_KEY} -lt 5 ]; then + error "API key appears to be too short (<5 chars)." + fi + + # Build the header argument for Admin API Authentication via unblock key + AUTH_HEADER_ARG="X-Dataverse-unblock-key: ${ADMIN_API_UNBLOCK_KEY}" +fi + +# Check for file with DB options given, file present and readable as well as parseable by yq +# If parseable, render as JSON to temp file +CONV_CONF_FILE=$(mktemp) +if [ -f "${CONFIG_FILE}" ] && [ -r "${CONFIG_FILE}" ]; then + yq -M -o json "${CONFIG_FILE}" > "${CONV_CONF_FILE}" || error "Could not parse config file with yq from ${CONFIG_FILE}." +else + error "Could not read a config file at ${CONFIG_FILE}." +fi + +# Check or wait for Dataverse API being responsive +echo "Waiting for ${DATAVERSE_URL} to become ready in max ${TIMEOUT}." +wait4x http "${DATAVERSE_URL}/api/info/version" -i 8s -t "$TIMEOUT" --expect-status-code 200 --expect-body-json data.version + +# Check for Dataverse Admin API endpoints being reachable by retrieving the current DB options, expect blockades! +CURRENT_SETTINGS=$(mktemp) +echo "Retrieving settings from running instance." +# TODO: Do we need to support pre v6.7 style unblock key query parameter? +curl -sSL --fail-with-body -o "${CURRENT_SETTINGS}" -H "${AUTH_HEADER_ARG}" "${DATAVERSE_URL}/api/admin/settings" \ + || error "Failed. Response message: $( cat "${CURRENT_SETTINGS}")" \ + && echo "Success!" + # TODO: while it's nice to have the current settings written out, it may contain sensitive information (so don't). + # && ( echo "Success! Current settings: "; jq '.data' < "$CURRENT_SETTINGS" ) + +# We need to make the settings update atomic. +echo "Replacing settings." +RESPONSE=$(mktemp) +curl -sSL --fail-with-body -o "${RESPONSE}" -X PUT -H "${AUTH_HEADER_ARG}" --json @"${CONV_CONF_FILE}" "${DATAVERSE_URL}/api/admin/settings" \ + || error "Failed. Response message: $( jq ".message" < "${RESPONSE}" )" \ + && ( echo -e "Success!\nOperations executed: "; jq '.data' < "$RESPONSE" ) From 558e8f98992d7f8fddafeb8fc975e8305a7b6b23 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 25 Jul 2025 12:33:12 +0200 Subject: [PATCH 255/634] refactor(ct): extract common utility functions in `apply-db-settings.sh` to `common.sh` #11639 - Moved shared functions (`error`, `exists_on_path`, `require_on_path`) to a new reusable script `common.sh`. - Simplified `apply-db-settings.sh` by sourcing `common.sh` for dependency validation logic. --- .../scripts/apply-db-settings.sh | 24 ++++--------------- .../scripts/util/common.sh | 17 +++++++++++++ 2 files changed, 22 insertions(+), 19 deletions(-) create mode 100644 modules/container-configbaker/scripts/util/common.sh diff --git a/modules/container-configbaker/scripts/apply-db-settings.sh b/modules/container-configbaker/scripts/apply-db-settings.sh index 0d0a2ecedbc..65b9639c07e 100755 --- a/modules/container-configbaker/scripts/apply-db-settings.sh +++ b/modules/container-configbaker/scripts/apply-db-settings.sh @@ -25,29 +25,15 @@ function usage() { exit 1 } -### Common functions -function error { - echo "ERROR:" "$@" >&2 - exit 2 -} - -function exists { - type "$1" >/dev/null 2>&1 && return 0 - ( IFS=:; for p in $PATH; do [ -x "${p%/}/$1" ] && return 0; done; return 1 ) -} +source util/common.sh # Check for (the right) yq, jq, and wait4x being available -if ! exists yq; then - error "No yq executable found on PATH." -elif ! grep -q "https://github.com/mikefarah/yq" <((yq --version)); then +require_on_path yq +if ! grep -q "https://github.com/mikefarah/yq" <((yq --version)); then error "You must install yq from https://github.com/mikefarah/yq, not https://github.com/kislyuk/yq" fi -if ! exists jq; then - error "No jq executable found on PATH." -fi -if ! exists wait4x; then - error "No wait4x executable found on PATH." -fi +require_on_path jq +require_on_path wait4x # Set some defaults as documented DATAVERSE_URL=${DATAVERSE_URL:-"http://dataverse:8080"} diff --git a/modules/container-configbaker/scripts/util/common.sh b/modules/container-configbaker/scripts/util/common.sh new file mode 100644 index 00000000000..91de5257a5c --- /dev/null +++ b/modules/container-configbaker/scripts/util/common.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +function error { + echo "ERROR:" "$@" >&2 + exit 2 +} + +function exists_on_path { + type "$1" >/dev/null 2>&1 && return 0 + ( IFS=:; for p in $PATH; do [ -x "${p%/}/$1" ] && return 0 || echo "${p%/}/$1"; done; return 1 ) +} + +function require_on_path { + if ! exists_on_path "$1"; then + error "No $1 executable found on PATH." + fi +} From 8e220fb674a0e15852ee9066a72b742ecc597382 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 25 Jul 2025 12:34:44 +0200 Subject: [PATCH 256/634] feat(ct): support additional environment variable sources in `apply-db-settings.sh` #11639 - Added `-e`/`ENV_SOURCE` option to specify a file or directory as a source for additional environment variables. - Introduced `read-to-env.sh` utility script to parse environment variables from specified sources. - Updated usage information and parsing logic in `apply-db-settings.sh` to handle the new option. --- .../scripts/apply-db-settings.sh | 21 ++++++++++-- .../scripts/util/read-to-env.sh | 34 +++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 modules/container-configbaker/scripts/util/read-to-env.sh diff --git a/modules/container-configbaker/scripts/apply-db-settings.sh b/modules/container-configbaker/scripts/apply-db-settings.sh index 65b9639c07e..c7e92ef42b0 100755 --- a/modules/container-configbaker/scripts/apply-db-settings.sh +++ b/modules/container-configbaker/scripts/apply-db-settings.sh @@ -5,7 +5,7 @@ set -euo pipefail function usage() { - echo "Usage: $(basename "$0") [-h] [-u instanceUrl] [-t timeout] [-c configFile] [-b unblockKey]" + echo "Usage: $(basename "$0") [-h] [-u instanceUrl] [-t timeout] [-c configFile] [-b unblockKey] [-e envSource]" echo "" echo "Replace all Database Settings in a running Dataverse installation in an idempotent way." echo "" @@ -15,9 +15,12 @@ function usage() { echo " timeout - Provide how long to wait for the instance to become available (using wait4x). Default: '3m'" echo " Can be set as environment variable 'TIMEOUT'." echo " configFile - Path to a JSON, YAML, PROPERTIES or TOML file containing your settings. Default: '/dv/db-opts.yml'" - echo " Can be set as environment variable 'CONFIG_FILE'." + echo " Can be set as environment variable 'CONFIG_FILE'. May contain \${var} references to env. vars." echo " unblockKey - Either string or path to a file with the Admin API Unblock Key. Optional for localhost. No default." echo " Can be set as environment variable 'ADMIN_API_UNBLOCK_KEY'." + echo " envSource - Path to a file or directory used as source for additional environment variables." + echo " Optional, no default. Can be set as environment variable 'ENV_SOURCE'." + echo " Environment variables from this file or directory structure will be script-local." echo "" echo "Note: This script will wait for the Dataverse instance to be available before executing the replacement." echo " Be careful - this script will not stop you from deleting any vital settings." @@ -26,6 +29,7 @@ function usage() { } source util/common.sh +source util/read-to-env.sh # Check for (the right) yq, jq, and wait4x being available require_on_path yq @@ -40,20 +44,31 @@ DATAVERSE_URL=${DATAVERSE_URL:-"http://dataverse:8080"} ADMIN_API_UNBLOCK_KEY=${ADMIN_API_UNBLOCK_KEY:-""} TIMEOUT=${TIMEOUT:-"3m"} CONFIG_FILE=${CONFIG_FILE:-"/dv/db-opts.yml"} +ENV_SOURCE=${ENV_SOURCE:-""} -while getopts "u:t:c:b:h" OPTION +while getopts "u:t:c:b:e:h" OPTION do case "$OPTION" in u) DATAVERSE_URL="$OPTARG" ;; t) TIMEOUT="$OPTARG" ;; c) CONFIG_FILE="$OPTARG" ;; b) ADMIN_API_UNBLOCK_KEY="$OPTARG" ;; + e) ENV_SOURCE="$OPTARG" ;; h) usage;; \?) usage;; esac done shift $((OPTIND-1)) +##### ##### ##### ##### ##### ##### ##### ##### ##### ##### ##### ##### ##### ##### ##### +# PARSE CONFIGURATION + +# In case the env source was given as cmd arg, parse it +if [ -n "$ENV_SOURCE" ]; then + read_to_env "$ENV_SOURCE" +fi + + # Define an auth header argument (enabling usage of different ways) AUTH_HEADER_ARG="" diff --git a/modules/container-configbaker/scripts/util/read-to-env.sh b/modules/container-configbaker/scripts/util/read-to-env.sh new file mode 100644 index 00000000000..344cb6e38bf --- /dev/null +++ b/modules/container-configbaker/scripts/util/read-to-env.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/common.sh" + +# Read from a target into environment variables. +# Parameters: $target +# Case A) If $target is a file, simply source it. +# Case B) If $target is a directory, parse dirs and files in it as variable names and file content as value +function read_to_env() { + local target="$1" + + if [ -f "$target" ] && [ -r "$target" ]; then + set -o allexport + # shellcheck disable=SC1090 + source "$target" + set +o allexport + elif [ -d "$target" ] && [ -r "$target" ] && [ -x "$target" ]; then + # Find all files + FILES=$( find "$target" -type f -printf '%P\n' ) + for FILE in $FILES; do + # Same as MPCONFIG does! + VARNAME=$( echo "$FILE" | tr '[:lower:]' '[:upper:]' | tr '/' '_' ) + VARVAL=$( cat "$target/$FILE") + + # Use printf to create the variable in global scope + printf -v "$VARNAME" '%s' "$VARVAL" + export "${VARNAME?}" + done + else + error "'$target' not a (readable) environment file or directory" + fi +} From bde0384241c56fca1b153c5ebd0f352d0237024d Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 25 Jul 2025 12:37:09 +0200 Subject: [PATCH 257/634] feat,refactor(ct): move yq parsing and add envsubst #11639 - Enhanced `apply-db-settings.sh` to support environment variable substitution while parsing the configuration file using `yq` feature `envsubst()`. - Improved wording of error messages for unreadable or unparseable configuration files. - Moved the conversion logic above all interaction with the instance, as we should be sure all env var substitutions fail early (at least before waiting for the instance to come up). --- .../scripts/apply-db-settings.sh | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/modules/container-configbaker/scripts/apply-db-settings.sh b/modules/container-configbaker/scripts/apply-db-settings.sh index c7e92ef42b0..0780dd47f1c 100755 --- a/modules/container-configbaker/scripts/apply-db-settings.sh +++ b/modules/container-configbaker/scripts/apply-db-settings.sh @@ -68,6 +68,19 @@ if [ -n "$ENV_SOURCE" ]; then read_to_env "$ENV_SOURCE" fi +# Check for file with DB options given, file present and readable as well as parseable by yq +# If parseable, render as JSON to temp file +CONV_CONF_FILE=$(mktemp) +if [ -f "${CONFIG_FILE}" ] && [ -r "${CONFIG_FILE}" ]; then + # See https://mikefarah.gitbook.io/yq/operators/env-variable-operators#tip + yq -M -o json '(.. | select(tag == "!!str")) |= envsubst(nu)' "${CONFIG_FILE}" > "${CONV_CONF_FILE}" || error "Could not parse config file with yq from '${CONFIG_FILE}'." + cat "$CONV_CONF_FILE" +else + error "Could not read a config file at '${CONFIG_FILE}'." +fi + +##### ##### ##### ##### ##### ##### ##### ##### ##### ##### ##### ##### ##### ##### ##### +# API INTERACTION # Define an auth header argument (enabling usage of different ways) AUTH_HEADER_ARG="" @@ -99,15 +112,6 @@ if ! [[ "${DATAVERSE_URL}" == *"://localhost"* ]] || [ -n "${ADMIN_API_UNBLOCK_K AUTH_HEADER_ARG="X-Dataverse-unblock-key: ${ADMIN_API_UNBLOCK_KEY}" fi -# Check for file with DB options given, file present and readable as well as parseable by yq -# If parseable, render as JSON to temp file -CONV_CONF_FILE=$(mktemp) -if [ -f "${CONFIG_FILE}" ] && [ -r "${CONFIG_FILE}" ]; then - yq -M -o json "${CONFIG_FILE}" > "${CONV_CONF_FILE}" || error "Could not parse config file with yq from ${CONFIG_FILE}." -else - error "Could not read a config file at ${CONFIG_FILE}." -fi - # Check or wait for Dataverse API being responsive echo "Waiting for ${DATAVERSE_URL} to become ready in max ${TIMEOUT}." wait4x http "${DATAVERSE_URL}/api/info/version" -i 8s -t "$TIMEOUT" --expect-status-code 200 --expect-body-json data.version From 2368d7eb924c9ccfeafeb94ca97adb07837efe6b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 1 Aug 2025 13:31:57 +0200 Subject: [PATCH 258/634] fix(migrate): move `UPDATE` statement post constraint drops for NULL handling Reordered `UPDATE` statement in migration script to occur after dropping constraints, ensuring NULL values in `lang` are handled correctly before setting NOT NULL constraints, but after we remove the initial constraints from v4.15. --- src/main/resources/db/migration/V6.7.0.2.sql | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/resources/db/migration/V6.7.0.2.sql b/src/main/resources/db/migration/V6.7.0.2.sql index 07a568f0834..a7493359ad5 100644 --- a/src/main/resources/db/migration/V6.7.0.2.sql +++ b/src/main/resources/db/migration/V6.7.0.2.sql @@ -3,10 +3,6 @@ -- 2. Update lang column to use empty string default instead of NULL (avoid non-unique pairs) -- 3. Add NOT NULL constraints and unique constraint for name+lang pairs --- First, update any existing NULL lang values to empty string -UPDATE setting SET lang = '' WHERE lang IS NULL; - --- Postgres doesn't support IF NOT EXISTS for ALTER COLUMN or ADD CONSTRAINT, so we need conditional logic DO $$ BEGIN -- These database constraints were added with Dataverse 4.15, but they had no representation in the code, @@ -16,7 +12,12 @@ BEGIN ALTER TABLE setting DROP CONSTRAINT IF EXISTS non_empty_lang; DROP INDEX IF EXISTS unique_settings; + -- Now, update any existing NULL lang values to empty string (we cannot do this before lifting the restrictions) + -- This also needs to be done before we try to alter the table to not allow NULL for setting.lang + UPDATE setting SET lang = '' WHERE lang IS NULL; + -- Only alter columns if they need to be changed + -- (Note: Postgres doesn't support IF NOT EXISTS for ALTER COLUMN or ADD CONSTRAINT, so we need conditional logic) IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'setting' AND column_name = 'name' AND (data_type = 'text' OR is_nullable = 'YES')) THEN From 21ca0b836c4abe027fe74e9d4e6d7d8c03f2c120 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 1 Aug 2025 16:36:02 +0200 Subject: [PATCH 259/634] fix(scripts): exclude hidden mounted files in `read-to-env.sh` Updated `read-to-env.sh` to handle Kubernetes secrets by excluding hidden mounted files and refining file detection logic with `find`. --- modules/container-configbaker/scripts/util/read-to-env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/container-configbaker/scripts/util/read-to-env.sh b/modules/container-configbaker/scripts/util/read-to-env.sh index 344cb6e38bf..485586521ab 100644 --- a/modules/container-configbaker/scripts/util/read-to-env.sh +++ b/modules/container-configbaker/scripts/util/read-to-env.sh @@ -17,8 +17,8 @@ function read_to_env() { source "$target" set +o allexport elif [ -d "$target" ] && [ -r "$target" ] && [ -x "$target" ]; then - # Find all files - FILES=$( find "$target" -type f -printf '%P\n' ) + # Find all files (K8s secrets are symlinks, so look for not directory & remove the hidden mounted files.) + FILES=$( find "$target" -not -type d -printf '%P\n' | grep -v '^\.\.' ) for FILE in $FILES; do # Same as MPCONFIG does! VARNAME=$( echo "$FILE" | tr '[:lower:]' '[:upper:]' | tr '/' '_' ) From 7f8329226cd21c5aa3480f8351fa069fbcec057e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 1 Aug 2025 17:35:01 +0200 Subject: [PATCH 260/634] chore(scripts): comment out config file print in `apply-db-settings.sh` #11639 Temporarily removed `cat` command for converted config file output. Added a TODO to consider a debug switch for conditional logging in the future. --- modules/container-configbaker/scripts/apply-db-settings.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/container-configbaker/scripts/apply-db-settings.sh b/modules/container-configbaker/scripts/apply-db-settings.sh index 0780dd47f1c..deb897d138c 100755 --- a/modules/container-configbaker/scripts/apply-db-settings.sh +++ b/modules/container-configbaker/scripts/apply-db-settings.sh @@ -74,7 +74,8 @@ CONV_CONF_FILE=$(mktemp) if [ -f "${CONFIG_FILE}" ] && [ -r "${CONFIG_FILE}" ]; then # See https://mikefarah.gitbook.io/yq/operators/env-variable-operators#tip yq -M -o json '(.. | select(tag == "!!str")) |= envsubst(nu)' "${CONFIG_FILE}" > "${CONV_CONF_FILE}" || error "Could not parse config file with yq from '${CONFIG_FILE}'." - cat "$CONV_CONF_FILE" + # TODO: think about adding a debug switch here, not just print + # cat "$CONV_CONF_FILE" else error "Could not read a config file at '${CONFIG_FILE}'." fi From 37015d6fcd55fca787930ca3da6887f87eab6edc Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 1 Aug 2025 18:17:36 +0200 Subject: [PATCH 261/634] fix(settings): handle JSON string unescaping in `SettingsServiceBean` #11639 Updated `SettingsServiceBean` to properly handle unescaping of JSON strings for string-type values. Enhanced tests in `SettingsServiceBeanTest` to validate behavior with new scenarios. --- .../settings/SettingsServiceBean.java | 16 +++++++++++----- .../settings/SettingsServiceBeanTest.java | 19 ++++++++++++------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index c29f94e2672..c555b059523 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -15,6 +15,7 @@ import jakarta.json.JsonArrayBuilder; import jakarta.json.JsonObject; import jakarta.json.JsonObjectBuilder; +import jakarta.json.JsonString; import jakarta.json.JsonValue; import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; @@ -1168,11 +1169,16 @@ static Set convertJsonToSettings(JsonObject settings) { return settings.entrySet().stream() .map(entry -> { String key = entry.getKey(); - String value = entry.getValue().toString() - // This is necessary to avoid storing the quotes inthe DB when a setting is a simple value. - // JsonValue will escape any JsonString with such quotes. - .replaceFirst("^\"", "") - .replaceFirst("\"$", ""); + + String value; + JsonValue jsonValue = entry.getValue(); + if (jsonValue.getValueType() == JsonValue.ValueType.STRING) { + // For string values, get the actual string content (unescaped) + value = ((JsonString) jsonValue).getString(); + } else { + // For objects, arrays, numbers, booleans, null - use JSON representation + value = jsonValue.toString(); + } if (key.contains(L10N_KEY_SEPARATOR)) { // Handle localized settings diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java index ed6d5417c06..eb4d67d1835 100644 --- a/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/settings/SettingsServiceBeanTest.java @@ -251,18 +251,23 @@ void testConvertJsonToSettings_simpleKeyValues() { // The REST API endpoint presents a JsonObject, which may have number literals in it. // Check that we can cope with that. .add(":Key3", 123456) + // Make sure we deal with quotes + .add(":Key4", " Dataverse © 2014-2025") .build(); // When Set result = SettingsServiceBean.convertJsonToSettings(input); // Then - assertEquals(3, result.size()); - assertEquals( - Set.of(new Setting(":Key1", "Value1"), - new Setting(":Key2", "123456"), - new Setting(":Key3", "123456") - ), result); + Map expectedResults = Map.of( + ":Key1", "Value1", + ":Key2", "123456", + ":Key3", "123456", + ":Key4", " Dataverse © 2014-2025" + ); + for (Setting setting : result) { + assertEquals(expectedResults.get(setting.getName()), setting.getContent()); + } } @Test @@ -277,7 +282,7 @@ void testConvertJsonToSettings_localizedKeysWithSimpleValues() { Set result = SettingsServiceBean.convertJsonToSettings(input); // Then - assertEquals(2, result.size()); + // Note: we do not verify the content with Setting.equals() - but we are not really interested in it as well. assertEquals( Set.of(new Setting(":LocalizedKey", "en", "EnglishValue"), new Setting(":LocalizedKey", "fr", "FrenchValue") From 292855bb0a9a9445e7628d4426b6a3103305cc14 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 1 Aug 2025 16:50:06 -0400 Subject: [PATCH 262/634] doc suggestions for settings API #11639 --- doc/sphinx-guides/source/api/changelog.rst | 10 + doc/sphinx-guides/source/api/native-api.rst | 186 +++++++++++++++--- .../source/installation/config.rst | 26 ++- 3 files changed, 187 insertions(+), 35 deletions(-) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 5be6c78adce..7a9bd446fb8 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -7,6 +7,16 @@ This API changelog is experimental and we would love feedback on its usefulness. :local: :depth: 1 +v6.9 +---- + +- The way to set per-format size limits for tabular ingest has changed. + JSON input is now used. + See :ref:`:TabularIngestSizeLimit`. +- In the past, the settings API would accept any key and value. + This is no longer the case because validation has been added. + See :ref:`settings_put_single`, for example. + v6.8 ---- diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index c55fc424496..a972894eb57 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -6777,61 +6777,191 @@ Admin This is the administrative part of the API. For security reasons, it is absolutely essential that you block it before allowing public access to a Dataverse installation. -Blocking can be done using settings. -See the ``post-install-api-block.sh`` script in the ``scripts/api`` folder for details. -See :ref:`blocking-api-endpoints` in Securing Your Installation section of the Configuration page of the Installation Guide. +See :ref:`blocking-api-endpoints` in the Installation Guide for details. + +.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. + +.. _admin-api-db-settings: + +Manage Database Settings +~~~~~~~~~~~~~~~~~~~~~~~~ + +These are the API endpoints for managing the :ref:`database-settings` listed in the Installation Guide. + +.. _settings_get_all: List All Database Settings -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash -List all settings:: + export SERVER_URL="http://localhost:8080" + + curl "$SERVER_URL/api/admin/settings" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl http://localhost:8080/api/admin/settings + +.. _settings_get_single: + +Get Single Database Setting +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + export SERVER_URL="http://localhost:8080" + export NAME=":UploadMethods" + + curl "$SERVER_URL/api/admin/settings/$NAME" + +The fully expanded example above (without environment variables) looks like this: - GET http://$SERVER/api/admin/settings +.. code-block:: bash + + curl http://localhost:8080/api/admin/settings/:UploadMethods + +.. _settings_get_single_lang: + +Get Single Database Setting With Language/Locale +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A small number of settings, most notably :ref:`:ApplicationTermsOfUse`, can be saved in multiple languages. + +Use two character language codes. + +.. code-block:: bash + + export SERVER_URL="http://localhost:8080" + export NAME=":ApplicationTermsOfUse" + export LANG="en" + + curl "$SERVER_URL/api/admin/settings/$NAME/lang/$LANG" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl http://localhost:8080/api/admin/settings/:ApplicationTermsOfUse/lang/en + +.. _settings_put_single: + +Configure Single Database Setting +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + export SERVER_URL="http://localhost:8080" + export NAME=":InstallationName" + export VALUE="LibreScholar" + + curl -X PUT "$SERVER_URL/api/admin/settings/$NAME" -d "$VALUE" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -X PUT http://localhost:8080/api/admin/settings/:InstallationName -d LibreScholar + +Note: ``NAME`` values are validated for existence and compliance. + +.. _settings_put_single_lang: + +Configure Single Database Setting With Language/Locale +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A small number of settings, most notably :ref:`:ApplicationTermsOfUse`, can be saved in multiple languages. + +Use two character language codes. + +.. code-block:: bash + + export SERVER_URL="http://localhost:8080" + export NAME=":ApplicationTermsOfUse" + export LANG="fr" + + curl -X PUT "$SERVER_URL/api/admin/settings/$NAME/lang/$LANG" --upload-file /tmp/apptou_fr.html + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -X PUT http://localhost:8080/api/admin/settings/:ApplicationTermsOfUse/lang/fr --upload-file /tmp/apptou_fr.html + +Note: ``NAME`` and ``LANG`` values are validated for existence and compliance. .. _settings_put_bulk: Configure All Database Settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Using a JSON file, replace all settings in a single idempotent and atomic operation and delete any settings not present in that JSON file. + +Use the JSON ``data`` object in output of ``GET /api/admin/settings`` (:ref:`settings_get_all`) for the JSON input structure for this endpoint. +To put this concretely, you can save just the ``data`` object for your existing settings to disk by filtering them through ``.jq`` like this: -Replace all settings in a single idempotent and atomic operation:: +.. code-block:: bash - PUT http://$SERVER/api/admin/settings + curl http://localhost:8080/api/admin/settings | jq '.data' > /tmp/all-settings.json -See JSON ``data`` object in output of ``GET /api/admin/settings`` for the JSON input structure for this endpoint. +Then you can use this "all-settings.json" file as a starting point for your input file. The :doc:`../installation/config` page of the Installation Guide has a :ref:`complete list of all the available settings `. +Note that settings in the JSON file are validated for existence and compliance. -Configure Single Database Setting -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. code-block:: bash -Sets setting ``name`` to the body of the request:: + export SERVER_URL="http://localhost:8080" + + curl -X PUT -H "Content-type:application/json" "$SERVER_URL/api/admin/settings" --upload-file /tmp/all-settings.json - PUT http://$SERVER/api/admin/settings/$name +The fully expanded example above (without environment variables) looks like this: -Sets a localized setting ``name`` for locale/language ``lang`` to the body of the request:: +.. code-block:: bash - PUT http://$SERVER/api/admin/settings/$name/lang/$lang + curl -X PUT -H "Content-type:application/json" http://localhost:8080/api/admin/settings --upload-file /tmp/all-settings.json -Get Single Database Setting -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. _settings_delete_single: -Get the setting under ``name``:: +Delete Single Database Setting +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - GET http://$SERVER/api/admin/settings/$name +.. code-block:: bash -Gets a localized setting under ``name`` for locale/language ``lang``:: + export SERVER_URL="http://localhost:8080" + export NAME=":InstallationName" + + curl -X DELETE "$SERVER_URL/api/admin/settings/$NAME" - GET http://$SERVER/api/admin/settings/$name/lang/$lang +The fully expanded example above (without environment variables) looks like this: -Delete Single Database Setting -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. code-block:: bash + + curl -X DELETE http://localhost:8080/api/admin/settings/:InstallationName + +.. _settings_delete_single_lang: + +Delete Single Database Setting With Language/Locale +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A small number of settings, most notably :ref:`:ApplicationTermsOfUse`, can be saved in multiple languages. -Delete the setting under ``name``:: +Use two character language codes. - DELETE http://$SERVER/api/admin/settings/$name +.. code-block:: bash + + export SERVER_URL="http://localhost:8080" + export NAME=":ApplicationTermsOfUse" + export LANG="fr" + + curl -X DELETE "$SERVER_URL/api/admin/settings/$NAME/lang/$LANG" -Delete a localized setting under ``name`` for locale/language ``lang``:: +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash - DELETE http://$SERVER/api/admin/settings/$name/lang/$lang + curl -X DELETE http://localhost:8080/api/admin/settings/:ApplicationTermsOfUse/lang/fr .. _list-all-feature-flags: diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index f064cc51a06..c7f76c04754 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3873,13 +3873,13 @@ You might also create your own profiles and use these, please refer to the upstr Database Settings ----------------- -These settings are stored in the ``setting`` database table but can be read and modified via the "admin" endpoint of the :doc:`/api/native-api` for easy scripting. +These settings are stored in the ``setting`` database table but we recommend using the Admin API (:ref:`admin-api-db-settings`) to view and modify them, as shown below. -The most commonly used configuration options are listed first. +In short: -The pattern you will observe in curl examples below is that an HTTP ``PUT`` is used to add or modify a setting. If you perform an HTTP ``GET`` (the default when using curl), the output will contain the value of the setting, if it has been set. You can also do a ``GET`` of all settings with ``curl http://localhost:8080/api/admin/settings`` which you may want to pretty-print by piping the output through a tool such as jq by appending ``| jq .``. If you want to remove a setting, use an HTTP ``DELETE`` such as ``curl -X DELETE http://localhost:8080/api/admin/settings/:GuidesBaseUrl`` . - -For your convenience, there is also an Admin API endpoint to :ref:`bulk manage database settings in an atomic, idempotent fashion `. +- HTTP ``GET`` is used to show settings. +- HTTP ``PUT`` is used to add or modify settings. +- HTTP ``DELETE`` is used to delete settings. .. _:BlockedApiPolicy: @@ -4374,10 +4374,14 @@ For performance reasons, your Dataverse installation will only allow creation of In the UI, users trying to download a zip file larger than the Dataverse installation's :ZipDownloadLimit will receive messaging that the zip file is too large, and the user will be presented with alternate access options. +.. _:TabularIngestSizeLimit: + :TabularIngestSizeLimit +++++++++++++++++++++++ Threshold in bytes for limiting whether or not "ingest" is attempted for an uploaded tabular file (which can be resource intensive). +For more on the ingest features, see :doc:`/user/tabulardataingest/index` in the User Guide. + For example, with the below in place, files greater than 2 GB in size will not go through the ingest process: ``curl -X PUT -d 2000000000 http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit`` @@ -4394,6 +4398,8 @@ Using a JSON-based setting, you can override this global setting on a per-format - CSV - XLSX +(In previous releases of Dataverse, a colon-separated form was used to specify per-format limits, such as ``:TabularIngestSizeLimit:Rdata``, but this is no longer supported. Now JSON is used.) + The JSON follows this form, all fields optional: .. code:: json @@ -4405,8 +4411,14 @@ The JSON follows this form, all fields optional: "formatZ": "100" } -The ``default`` key represents the global default (with it being absent meaning the implicit global default of ``-1`` applies). -Add a format name (as listed above) to change the limit for this particular format. +Whatever JSON you send will overwrite existing values. If you have any current settings, you can use the following command to see them in the proper format (and then add the new settings you want): + +``curl http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit | jq -r '.data.message'`` + +The ``default`` key is optional and can be used to give limits to formats that are not specified in the JSON. If you omit the ``default`` key or set it to ``-1``, no limits are applied to formats not specified in the JSON. + +Add a format name (DTA, POR, etc., as listed above) to change the limit for this particular format. + Any size limits must be provided as string literals (in quotes), not number literals! Examples: From b7e541d5cdfc2a4d0f15e3f7de81ddb565423217 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 4 Aug 2025 15:56:32 -0400 Subject: [PATCH 263/634] doc tweaks #11639 --- doc/sphinx-guides/source/api/native-api.rst | 8 ++++---- doc/sphinx-guides/source/installation/config.rst | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index a972894eb57..30fb4c24d70 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -6830,7 +6830,7 @@ Get Single Database Setting With Language/Locale A small number of settings, most notably :ref:`:ApplicationTermsOfUse`, can be saved in multiple languages. -Use two character language codes. +Use two-character ISO 639-1 language codes. .. code-block:: bash @@ -6874,7 +6874,7 @@ Configure Single Database Setting With Language/Locale A small number of settings, most notably :ref:`:ApplicationTermsOfUse`, can be saved in multiple languages. -Use two character language codes. +Use two-character ISO 639-1 language codes. .. code-block:: bash @@ -6900,7 +6900,7 @@ Configure All Database Settings Using a JSON file, replace all settings in a single idempotent and atomic operation and delete any settings not present in that JSON file. Use the JSON ``data`` object in output of ``GET /api/admin/settings`` (:ref:`settings_get_all`) for the JSON input structure for this endpoint. -To put this concretely, you can save just the ``data`` object for your existing settings to disk by filtering them through ``.jq`` like this: +To put this concretely, you can save just the ``data`` object for your existing settings to disk by filtering them through ``jq`` like this: .. code-block:: bash @@ -6947,7 +6947,7 @@ Delete Single Database Setting With Language/Locale A small number of settings, most notably :ref:`:ApplicationTermsOfUse`, can be saved in multiple languages. -Use two character language codes. +Use two-character ISO 639-1 language codes. .. code-block:: bash diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index c7f76c04754..7e4d85b360f 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -4389,7 +4389,7 @@ For example, with the below in place, files greater than 2 GB in size will not g You can set this value to ``0`` to prevent files from being ingested at all. The default is ``-1``, meaning no file size limit is applied. -Using a JSON-based setting, you can override this global setting on a per-format basis for the following formats: +Using a JSON-based setting, you can override this global setting on a per-format basis for the following formats (case-insensitive): - DTA - POR @@ -4417,13 +4417,13 @@ Whatever JSON you send will overwrite existing values. If you have any current s The ``default`` key is optional and can be used to give limits to formats that are not specified in the JSON. If you omit the ``default`` key or set it to ``-1``, no limits are applied to formats not specified in the JSON. -Add a format name (DTA, POR, etc., as listed above) to change the limit for this particular format. +Add a format name (DTA, POR, etc., as listed above) to change the limit for that particular format. Any size limits must be provided as string literals (in quotes), not number literals! Examples: -1. If you want your Dataverse installation to not attempt to ingest Rdata files larger than 1 MB but otherwise unlimited: +1. If you want your Dataverse installation to not attempt to ingest Rdata files larger than 1 MB but otherwise be unlimited: ``curl -X PUT -d '{"Rdata":"1000000"}' http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit`` 2. If you want your Dataverse installation to not attempt to ingest XLSX files at all and apply a global limit of 512 MiB, use this setting: From 071f695f13b6516aada232cac5e234407798f710 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 17 Sep 2025 08:26:26 +0200 Subject: [PATCH 264/634] chore: move migrations to be part of v6.9 instead of v6.8 #11639 --- src/main/resources/db/migration/{V6.7.0.1.sql => V6.8.0.1.sql} | 0 src/main/resources/db/migration/{V6.7.0.2.sql => V6.8.0.2.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V6.7.0.1.sql => V6.8.0.1.sql} (100%) rename src/main/resources/db/migration/{V6.7.0.2.sql => V6.8.0.2.sql} (100%) diff --git a/src/main/resources/db/migration/V6.7.0.1.sql b/src/main/resources/db/migration/V6.8.0.1.sql similarity index 100% rename from src/main/resources/db/migration/V6.7.0.1.sql rename to src/main/resources/db/migration/V6.8.0.1.sql diff --git a/src/main/resources/db/migration/V6.7.0.2.sql b/src/main/resources/db/migration/V6.8.0.2.sql similarity index 100% rename from src/main/resources/db/migration/V6.7.0.2.sql rename to src/main/resources/db/migration/V6.8.0.2.sql From efd7bbfc3096479eb2a02cd32704a719b6385200 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 5 Aug 2025 16:25:48 -0400 Subject: [PATCH 265/634] create test methods for getting and setting all settings (and use them) #11639 --- .../edu/harvard/iq/dataverse/api/AdminIT.java | 39 ++++--------------- .../edu/harvard/iq/dataverse/api/UtilIT.java | 14 +++++++ 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java index 83a62d41b9f..6f3ffaa83b8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java @@ -80,9 +80,7 @@ void testSettingsRoundTrip() { UtilIT.setSetting(harmlessSetting, harmlessL10nValue, language); // Step 1: Get current settings state - Response getResponse = given() - .when() - .get("/api/admin/settings"); + Response getResponse = UtilIT.getSettings(); getResponse.then() .assertThat() @@ -111,12 +109,7 @@ void testSettingsRoundTrip() { .body("data.message", equalTo(harmlessValue)); // Step 4: Put back the original settings (this is what we're testing) - Response putResponse = given() - //.header("X-Dataverse-key", "") - .header("Content-Type", "application/json") - .body(originalSettings.toString()) - .when() - .put("/api/admin/settings"); + Response putResponse = UtilIT.setSettings(originalSettings.toString()); putResponse.then() .assertThat() @@ -125,20 +118,14 @@ void testSettingsRoundTrip() { .body("message.message", containsString("successfully updated")); // Step 5: Verify the harmless setting is gone (restored to original state) - Response verifyRestoredResponse = given() - //.header("X-Dataverse-key", "") - .when() - .get("/api/admin/settings" + harmlessSetting.toString()); + Response verifyRestoredResponse = UtilIT.getSetting(harmlessSetting); verifyRestoredResponse.then() .assertThat() .statusCode(NOT_FOUND.getStatusCode()); // Should not exist anymore // Step 6: Verify overall settings state matches original - Response finalGetResponse = given() - //.header("X-Dataverse-key", "") - .when() - .get("/api/admin/settings"); + Response finalGetResponse = UtilIT.getSettings(); finalGetResponse.then() .assertThat() @@ -165,9 +152,7 @@ void testGetAllSettingsWithLocalization() { UtilIT.setSetting(harmlessSetting, harmlessL10nValue, language); // When - Response getResponse = given() - .when() - .get("/api/admin/settings"); + Response getResponse = UtilIT.getSettings(); // Then getResponse.then() @@ -184,11 +169,7 @@ void testGetAllSettingsWithLocalization() { @Test void testPutAllSettingsWithEmptyJson() { // Test error handling for empty JSON - Response response = given() - .header("Content-Type", "application/json") - .body("{}") - .when() - .put("/api/admin/settings"); + Response response = UtilIT.setSettings("{}"); response.then() .assertThat() @@ -199,11 +180,7 @@ void testPutAllSettingsWithEmptyJson() { @Test void testPutAllSettingsWithInvalidSetting() { // Test error handling for empty JSON - Response response = given() - .header("Content-Type", "application/json") - .body("{\":Test1\": \"Foobar\", \":Test2\": \"Foobar\" }") - .when() - .put("/api/admin/settings"); + Response response = UtilIT.setSettings("{\":Test1\": \"Foobar\", \":Test2\": \"Foobar\" }"); response.then() .assertThat() @@ -242,7 +219,7 @@ public void testListAuthenticatedUsers() throws Exception { Response deleteSuperuser = UtilIT.deleteUser(superuserUsername); assertEquals(200, deleteSuperuser.getStatusCode()); - } +} @Test public void testFilterAuthenticatedUsersForbidden() throws Exception { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index aa11ceb0d18..548627554d2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2507,6 +2507,11 @@ static Response deleteSetting(String settingKey) { return response; } + static Response getSettings() { + Response response = given().when().get("/api/admin/settings"); + return response; + } + static Response getSetting(SettingsServiceBean.Key settingKey) { Response response = given().when().get("/api/admin/settings/" + settingKey); return response; @@ -2535,6 +2540,15 @@ public static Response setSetting(String settingKey, String value) { return response; } + public static Response setSettings(String value) { + Response response = given() + .header("Content-Type", "application/json") + .body(value) + .when() + .put("/api/admin/settings"); + return response; + } + static Response getFeatureFlags() { Response response = given().when().get("/api/admin/featureFlags"); return response; From 95533c68486dc8fcb5451c23f7d935c8b268bb7e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 7 Aug 2025 13:10:27 -0400 Subject: [PATCH 266/634] the "not found" message now ends in a period #11639 Make the test comply with the change. --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index b273502e6ed..6d160c038ff 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -2401,7 +2401,7 @@ public void testFileChecksum() { Response getDefaultSetting = UtilIT.getSetting(SettingsServiceBean.Key.FileFixityChecksumAlgorithm); getDefaultSetting.prettyPrint(); getDefaultSetting.then().assertThat() - .body("message", equalTo("Setting :FileFixityChecksumAlgorithm not found")); + .body("message", equalTo("Setting :FileFixityChecksumAlgorithm not found.")); Response uploadMd5File = UtilIT.uploadRandomFile(dataset1PersistentId, apiToken); uploadMd5File.prettyPrint(); From 30f4603521154428a22ff55665f99239b284345d Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 7 Aug 2025 15:30:57 -0400 Subject: [PATCH 267/634] add to the enum many archive-related settings #11639 Attempts to set these settings were failing with a 400 and "The name of the setting is invalid." By adding them to the enum, they pass validation and are able to work again. These settings are really subsettings of :ArchiverSettings. (See installation/config.rst.) The exception is :BagGeneratorThreads, which is not a subsetting of :ArchiverSettings. Rather, it is a related setting used by multiple archivers. :DRSArchiverConfig was added to the enum. DRS is a system specific to Harvard which is why we don't document the setting in the guides, but we plan to mention it in the release notes. See also https://github.com/IQSS/dataverse.harvard.edu/issues/177 --- .../11639-db-opts-idempotency.md | 14 ++++++++ .../impl/DRSSubmitToArchiveCommand.java | 3 +- .../impl/DuraCloudSubmitToArchiveCommand.java | 9 ++++-- .../GoogleCloudSubmitToArchiveCommand.java | 6 ++-- .../impl/LocalSubmitToArchiveCommand.java | 3 +- .../impl/S3SubmitToArchiveCommand.java | 3 +- .../settings/SettingsServiceBean.java | 32 +++++++++++++++++++ .../iq/dataverse/util/bagit/BagGenerator.java | 3 +- .../edu/harvard/iq/dataverse/api/BagIT.java | 8 +++-- 9 files changed, 70 insertions(+), 11 deletions(-) diff --git a/doc/release-notes/11639-db-opts-idempotency.md b/doc/release-notes/11639-db-opts-idempotency.md index 2af82f4ffda..2a87a81eff8 100644 --- a/doc/release-notes/11639-db-opts-idempotency.md +++ b/doc/release-notes/11639-db-opts-idempotency.md @@ -19,3 +19,17 @@ As an administrator of a Dataverse instance, you can now make use of enhanced Bu It allows control over all Database Settings from a single source without risking an undefined state. Note: Despite the validation of setting names and languages, the content of any database setting is still not being validated when using the Settings Admin API! + +### Updated Database Settings + +The following database settings are were added to the official list within the code (to remain valid with the settings cleanup mentioned above): + +- :BagGeneratorThreads +- :BagItLocalPath +- :BuiltinUsersKey - formerly BuiltinUsers.KEY +- :DRSArchiverConfig - a Harvard-specific setting +- :DuraCloudContext +- :DuraCloudHost +- :DuraCloudPort +- :GoogleCloudBucket +- :GoogleCloudProject diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DRSSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DRSSubmitToArchiveCommand.java index 594d4fe25ba..78e8454255b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DRSSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DRSSubmitToArchiveCommand.java @@ -56,12 +56,13 @@ import com.auth0.jwt.JWT; import com.auth0.jwt.algorithms.Algorithm; import com.auth0.jwt.exceptions.JWTCreationException; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.DRSArchiverConfig; @RequiredPermissions(Permission.PublishDataset) public class DRSSubmitToArchiveCommand extends S3SubmitToArchiveCommand implements Command { private static final Logger logger = Logger.getLogger(DRSSubmitToArchiveCommand.class.getName()); - private static final String DRS_CONFIG = ":DRSArchiverConfig"; + private static final String DRS_CONFIG = DRSArchiverConfig.toString(); private static final String ADMIN_METADATA = "admin_metadata"; private static final String S3_BUCKET_NAME = "s3_bucket_name"; private static final String S3_PATH = "s3_path"; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java index 94f983f0c13..fe4a25091d7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java @@ -7,6 +7,9 @@ import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.DuraCloudContext; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.DuraCloudHost; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.DuraCloudPort; import edu.harvard.iq.dataverse.workflow.step.Failure; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult; @@ -36,9 +39,9 @@ public class DuraCloudSubmitToArchiveCommand extends AbstractSubmitToArchiveComm private static final Logger logger = Logger.getLogger(DuraCloudSubmitToArchiveCommand.class.getName()); private static final String DEFAULT_PORT = "443"; private static final String DEFAULT_CONTEXT = "durastore"; - private static final String DURACLOUD_PORT = ":DuraCloudPort"; - private static final String DURACLOUD_HOST = ":DuraCloudHost"; - private static final String DURACLOUD_CONTEXT = ":DuraCloudContext"; + private static final String DURACLOUD_PORT = DuraCloudPort.toString(); + private static final String DURACLOUD_HOST = DuraCloudHost.toString(); + private static final String DURACLOUD_CONTEXT = DuraCloudContext.toString(); public DuraCloudSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GoogleCloudSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GoogleCloudSubmitToArchiveCommand.java index 7d749262b87..7dfb9f07e19 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GoogleCloudSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GoogleCloudSubmitToArchiveCommand.java @@ -14,6 +14,8 @@ import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.settings.JvmSettings; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.GoogleCloudBucket; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.GoogleCloudProject; import edu.harvard.iq.dataverse.workflow.step.Failure; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult; import org.apache.commons.codec.binary.Hex; @@ -35,8 +37,8 @@ public class GoogleCloudSubmitToArchiveCommand extends AbstractSubmitToArchiveCommand { private static final Logger logger = Logger.getLogger(GoogleCloudSubmitToArchiveCommand.class.getName()); - private static final String GOOGLECLOUD_BUCKET = ":GoogleCloudBucket"; - private static final String GOOGLECLOUD_PROJECT = ":GoogleCloudProject"; + private static final String GOOGLECLOUD_BUCKET = GoogleCloudBucket.toString(); + private static final String GOOGLECLOUD_PROJECT = GoogleCloudProject.toString(); public GoogleCloudSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version) { super(aRequest, version); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java index d2f061b6e70..462879f2ec9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java @@ -8,6 +8,7 @@ import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.BagItLocalPath; import edu.harvard.iq.dataverse.util.bagit.BagGenerator; import edu.harvard.iq.dataverse.util.bagit.OREMap; import edu.harvard.iq.dataverse.workflow.step.Failure; @@ -38,7 +39,7 @@ public LocalSubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion ver public WorkflowStepResult performArchiveSubmission(DatasetVersion dv, ApiToken token, Map requestedSettings) { logger.fine("In LocalCloudSubmitToArchive..."); - String localPath = requestedSettings.get(":BagItLocalPath"); + String localPath = requestedSettings.get(BagItLocalPath.toString()); String zipName = null; //Set a failure status that will be updated if we succeed diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/S3SubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/S3SubmitToArchiveCommand.java index 4f93e88de5e..65531d775c8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/S3SubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/S3SubmitToArchiveCommand.java @@ -7,6 +7,7 @@ import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.S3ArchiverConfig; import edu.harvard.iq.dataverse.util.bagit.BagGenerator; import edu.harvard.iq.dataverse.util.bagit.OREMap; import edu.harvard.iq.dataverse.util.json.JsonUtil; @@ -64,7 +65,7 @@ public class S3SubmitToArchiveCommand extends AbstractSubmitToArchiveCommand { private ManagedExecutorService executorService; private static final Logger logger = Logger.getLogger(S3SubmitToArchiveCommand.class.getName()); - private static final String S3_CONFIG = ":S3ArchiverConfig"; + private static final String S3_CONFIG = S3ArchiverConfig.toString(); private static final Config config = ConfigProvider.getConfig(); protected S3AsyncClient s3 = null; diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index c555b059523..b97de383391 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -466,7 +466,39 @@ Whether Harvesting (OAI) service is enabled */ ArchiverClassName, + /** + * Custom settings for each archiver. See list below. + */ ArchiverSettings, + /** + * :ArchiverSettings used by DRSSubmitToArchiveCommand. DRS is a system + * specific to Harvard which is why we don't document it in the guides. + * See also https://github.com/IQSS/dataverse.harvard.edu/issues/177 + */ + DRSArchiverConfig, + /** + * :ArchiverSettings used by DuraCloudSubmitToArchiveCommand. + */ + DuraCloudPort, + DuraCloudHost, + DuraCloudContext, + /** + * :ArchiverSettings used by GoogleCloudSubmitToArchiveCommand. + */ + GoogleCloudBucket, + GoogleCloudProject, + /** + * :ArchiverSettings used by LocalSubmitToArchiveCommand. + */ + BagItLocalPath, + /** + * :ArchiverSettings used by S3SubmitToArchiveCommand. + */ + S3ArchiverConfig, + /** + * :ArchiverSettings used by multiple archive commands. + */ + BagGeneratorThreads, /** * A comma-separated list of roles for which new dataverses should inherit the * corresponding role assignments from the parent dataverse. Also affects diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java index f6b12d5f904..f24ebdb8655 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java @@ -75,6 +75,7 @@ import edu.harvard.iq.dataverse.DataFile.ChecksumType; import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.settings.JvmSettings; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.BagGeneratorThreads; import edu.harvard.iq.dataverse.util.json.JsonLDTerm; import java.util.Optional; @@ -120,7 +121,7 @@ public class BagGenerator { private boolean usetemp = false; private int numConnections = 8; - public static final String BAG_GENERATOR_THREADS = ":BagGeneratorThreads"; + public static final String BAG_GENERATOR_THREADS = BagGeneratorThreads.toString(); private OREMap oremap; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java b/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java index c80e321b228..16c44003f35 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java @@ -2,6 +2,8 @@ import edu.harvard.iq.dataverse.engine.command.impl.LocalSubmitToArchiveCommand; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.BagGeneratorThreads; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.BagItLocalPath; import io.restassured.RestAssured; import static io.restassured.RestAssured.given; import io.restassured.response.Response; @@ -36,11 +38,13 @@ public static void setUpClass() { setArchiverClassName.then().assertThat() .statusCode(OK.getStatusCode()); - Response setArchiverSettings = UtilIT.setSetting(SettingsServiceBean.Key.ArchiverSettings, ":BagItLocalPath, :BagGeneratorThreads"); + // BagGeneratorThreads isn't used. Consider setting it or removing it. + Response setArchiverSettings = UtilIT.setSetting(SettingsServiceBean.Key.ArchiverSettings, + String.join(", ", BagItLocalPath.toString(), BagGeneratorThreads.toString())); setArchiverSettings.then().assertThat() .statusCode(OK.getStatusCode()); - Response setBagItLocalPath = UtilIT.setSetting(":BagItLocalPath", bagitExportDir); + Response setBagItLocalPath = UtilIT.setSetting(BagItLocalPath.toString(), bagitExportDir); setBagItLocalPath.then().assertThat() .statusCode(OK.getStatusCode()); From 4d72fcbf32b1e014913d87fed9b9e33c4f228dca Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 8 Aug 2025 10:19:08 -0400 Subject: [PATCH 268/634] add basic test for :TabularIngestSizeLimit #11639 --- .../edu/harvard/iq/dataverse/api/FilesIT.java | 80 ++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 38d89f782dd..7848314a834 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -60,6 +60,8 @@ public static void setUpClass() { Response removePublicInstall = UtilIT.deleteSetting(SettingsServiceBean.Key.PublicInstall); removePublicInstall.then().assertThat().statusCode(200); + Response removeLimit = UtilIT.deleteSetting(SettingsServiceBean.Key.TabularIngestSizeLimit); + removeLimit.then().assertThat().statusCode(OK.getStatusCode()); } @AfterAll @@ -1208,7 +1210,83 @@ public void test_AddFileBadUploadFormat() { } } - + + @Test + public void testIngestSizeLimits() throws InterruptedException, IOException { + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + Response makeSuperUser = UtilIT.setSuperuserStatus(username, true); + makeSuperUser.then().assertThat().statusCode(OK.getStatusCode()); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + + String tinyCsvOnly = """ +{ + "csv": "50" +} +"""; + + Response setLimit = UtilIT.setSetting(SettingsServiceBean.Key.TabularIngestSizeLimit, tinyCsvOnly); + setLimit.then().assertThat().statusCode(OK.getStatusCode()); + + Path pathToDataFile = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "data.csv"); + String contentOfCsv = "" + + "name,pounds,species,treats\n" + + "Midnight,15,dog,milkbones\n" + + "Tiger,17,cat,cat grass\n" + + "Panther,21,cat,cat nip\n"; + java.nio.file.Files.write(pathToDataFile, contentOfCsv.getBytes()); + + Response uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToDataFile.toString(), apiToken); + uploadFile.prettyPrint(); + uploadFile.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.files[0].label", equalTo("data.csv")); + + String fileId1 = JsonPath.from(uploadFile.body().asString()).getString("data.files[0].dataFile.id"); + + Response getFileDataTablesForNonTabularFileResponse = UtilIT.getFileDataTables(fileId1, apiToken); + getFileDataTablesForNonTabularFileResponse.prettyPrint(); + getFileDataTablesForNonTabularFileResponse.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo(BundleUtil.getStringFromBundle("files.api.only.tabular.supported"))); + + String largerCsv = """ +{ + "csv": "123456" +} +"""; + + setLimit = UtilIT.setSetting(SettingsServiceBean.Key.TabularIngestSizeLimit, largerCsv); + setLimit.then().assertThat().statusCode(OK.getStatusCode()); + + uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToDataFile.toString(), apiToken); + uploadFile.prettyPrint(); + uploadFile.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.files[0].label", equalTo("data-1.csv")); + + assertTrue(UtilIT.sleepForLock(datasetId.longValue(), "Ingest", apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION), "Failed test if Ingest Lock exceeds max duration " + pathToDataFile); + + String fileId2 = JsonPath.from(uploadFile.body().asString()).getString("data.files[0].dataFile.id"); + + Response getFileDataTablesForTabularFileResponse = UtilIT.getFileDataTables(fileId2, apiToken); + getFileDataTablesForTabularFileResponse.prettyPrint(); + getFileDataTablesForTabularFileResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].varQuantity", equalTo(4)); + + Response removeLimit = UtilIT.deleteSetting(SettingsServiceBean.Key.TabularIngestSizeLimit); + removeLimit.then().assertThat().statusCode(OK.getStatusCode()); + } + @Test public void testUningestFileViaApi() throws InterruptedException { Response createUser = UtilIT.createRandomUser(); From 159b189b417d38be4930dcbb6662e081c3fd8f0c Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 8 Aug 2025 12:19:47 -0400 Subject: [PATCH 269/634] more ingest size limit tests #11639 --- .../edu/harvard/iq/dataverse/api/FilesIT.java | 113 +++++++++++++++++- 1 file changed, 107 insertions(+), 6 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 7848314a834..c0b8543ff09 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -67,6 +67,7 @@ public static void setUpClass() { @AfterAll public static void tearDownClass() { UtilIT.deleteSetting(SettingsServiceBean.Key.PublicInstall); + UtilIT.deleteSetting(SettingsServiceBean.Key.TabularIngestSizeLimit); } /** @@ -1252,9 +1253,9 @@ public void testIngestSizeLimits() throws InterruptedException, IOException { String fileId1 = JsonPath.from(uploadFile.body().asString()).getString("data.files[0].dataFile.id"); - Response getFileDataTablesForNonTabularFileResponse = UtilIT.getFileDataTables(fileId1, apiToken); - getFileDataTablesForNonTabularFileResponse.prettyPrint(); - getFileDataTablesForNonTabularFileResponse.then().assertThat() + Response getTabularFails = UtilIT.getFileDataTables(fileId1, apiToken); + getTabularFails.prettyPrint(); + getTabularFails.then().assertThat() .statusCode(BAD_REQUEST.getStatusCode()) .body("message", equalTo(BundleUtil.getStringFromBundle("files.api.only.tabular.supported"))); @@ -1277,9 +1278,109 @@ public void testIngestSizeLimits() throws InterruptedException, IOException { String fileId2 = JsonPath.from(uploadFile.body().asString()).getString("data.files[0].dataFile.id"); - Response getFileDataTablesForTabularFileResponse = UtilIT.getFileDataTables(fileId2, apiToken); - getFileDataTablesForTabularFileResponse.prettyPrint(); - getFileDataTablesForTabularFileResponse.then().assertThat() + Response getTabularWorks = UtilIT.getFileDataTables(fileId2, apiToken); + getTabularWorks.prettyPrint(); + getTabularWorks.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].varQuantity", equalTo(4)); + + String tinyDefaultSize = """ +{ + "default": "50" +} +"""; + + setLimit = UtilIT.setSetting(SettingsServiceBean.Key.TabularIngestSizeLimit, tinyDefaultSize); + setLimit.then().assertThat().statusCode(OK.getStatusCode()); + + uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToDataFile.toString(), apiToken); + uploadFile.prettyPrint(); + uploadFile.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.files[0].label", equalTo("data-2.csv")); + + String fileId3 = JsonPath.from(uploadFile.body().asString()).getString("data.files[0].dataFile.id"); + + getTabularFails = UtilIT.getFileDataTables(fileId3, apiToken); + getTabularFails.prettyPrint(); + getTabularFails.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo(BundleUtil.getStringFromBundle("files.api.only.tabular.supported"))); + + // The behavior of `"default": "-2"` is not documented in the guides + // but it acts like `"default": "0"` which disables ingest. + String unexpectedNegativeDefault = """ +{ + "default": "-2" +} +"""; + + setLimit = UtilIT.setSetting(SettingsServiceBean.Key.TabularIngestSizeLimit, unexpectedNegativeDefault); + setLimit.then().assertThat().statusCode(OK.getStatusCode()); + + uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToDataFile.toString(), apiToken); + uploadFile.prettyPrint(); + uploadFile.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.files[0].label", equalTo("data-3.csv")); + + String fileId4 = JsonPath.from(uploadFile.body().asString()).getString("data.files[0].dataFile.id"); + + getTabularFails = UtilIT.getFileDataTables(fileId4, apiToken); + getTabularFails.prettyPrint(); + getTabularFails.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo(BundleUtil.getStringFromBundle("files.api.only.tabular.supported"))); + + // As the guides say, you MUST provide a string, not a JSON number. + // That is, `"123"` in quotes rather than `123` with no quotes. + // If you provide a number (no quotes) rather than a string, + // all ingest will be disabled and you'll see an error in server.log + // about how the system is misconfigured. + String invalidNonString = """ +{ + "default": 987654321 +} +"""; + + setLimit = UtilIT.setSetting(SettingsServiceBean.Key.TabularIngestSizeLimit, invalidNonString); + setLimit.then().assertThat().statusCode(OK.getStatusCode()); + + uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToDataFile.toString(), apiToken); + uploadFile.prettyPrint(); + uploadFile.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.files[0].label", equalTo("data-4.csv")); + + String fileId5 = JsonPath.from(uploadFile.body().asString()).getString("data.files[0].dataFile.id"); + + getTabularFails = UtilIT.getFileDataTables(fileId5, apiToken); + getTabularFails.prettyPrint(); + getTabularFails.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo(BundleUtil.getStringFromBundle("files.api.only.tabular.supported"))); + + String defaultDisabledAndLargeCsvLimit = """ +{ + "default": "0", + "csv": "123456" +} +"""; + + setLimit = UtilIT.setSetting(SettingsServiceBean.Key.TabularIngestSizeLimit, defaultDisabledAndLargeCsvLimit); + setLimit.then().assertThat().statusCode(OK.getStatusCode()); + + uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToDataFile.toString(), apiToken); + uploadFile.prettyPrint(); + uploadFile.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.files[0].label", equalTo("data-5.csv")); + + String fileId6 = JsonPath.from(uploadFile.body().asString()).getString("data.files[0].dataFile.id"); + + getTabularWorks = UtilIT.getFileDataTables(fileId2, apiToken); + getTabularWorks.prettyPrint(); + getTabularWorks.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].varQuantity", equalTo(4)); From 839dd79fa2a4d390c009ac4e02ffbf5f043419e3 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 8 Aug 2025 12:20:11 -0400 Subject: [PATCH 270/634] improve docs based on ingest size limit testing #11639 --- .../source/installation/config.rst | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 7e4d85b360f..f33c04eada8 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -4380,46 +4380,47 @@ In the UI, users trying to download a zip file larger than the Dataverse install +++++++++++++++++++++++ Threshold in bytes for limiting whether or not "ingest" is attempted for an uploaded tabular file (which can be resource intensive). -For more on the ingest features, see :doc:`/user/tabulardataingest/index` in the User Guide. +For more on the ingest feature, see :doc:`/user/tabulardataingest/index` in the User Guide. -For example, with the below in place, files greater than 2 GB in size will not go through the ingest process: +There are two ways to specify ingest size limits. You can set a global limit for all file types or you can use a JSON file for more granularity. We'll cover the global limit first. + +With the following value in place (again, expressed in bytes), files greater than 2 GB in size will not go through the ingest process: ``curl -X PUT -d 2000000000 http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit`` You can set this value to ``0`` to prevent files from being ingested at all. -The default is ``-1``, meaning no file size limit is applied. -Using a JSON-based setting, you can override this global setting on a per-format basis for the following formats (case-insensitive): +Out of the box, the ``:TabularIngestSizeLimit`` setting is absent, which results in ingest being attempted no matter how large the file is. You can specify this "no size limit" default explicitly with the value ``-1``. + +Using a JSON-based setting, you can set a global default and per-format limits for the following formats: +- CSV - DTA - POR -- SAV - Rdata -- CSV +- SAV - XLSX (In previous releases of Dataverse, a colon-separated form was used to specify per-format limits, such as ``:TabularIngestSizeLimit:Rdata``, but this is no longer supported. Now JSON is used.) -The JSON follows this form, all fields optional: +The expected JSON is an object with key/value pairs like the following. Format names are case-insensitive, and all fields are optional. The size limits must be strings with double quotes around them (e.g. ``"10"``) rather than numbers (e.g. ``10``). .. code:: json { "default": "-1", - "formatX": "0", - "formatY": "10", - "formatZ": "100" + "csv": "0", + "dta": "10", + "por": "100" } -Whatever JSON you send will overwrite existing values. If you have any current settings, you can use the following command to see them in the proper format (and then add the new settings you want): +Whatever JSON you send will overwrite existing values. If you have any exiting ``:TabularIngestSizeLimit`` settings, you can use the following command to see them in the expected input format above (and then add the new settings you want): ``curl http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit | jq -r '.data.message'`` -The ``default`` key is optional and can be used to give limits to formats that are not specified in the JSON. If you omit the ``default`` key or set it to ``-1``, no limits are applied to formats not specified in the JSON. +The ``default`` key is optional and can be used to give limits to formats that are not specified in the JSON. If you omit the ``default`` key or set it to ``"-1"``, no limits are applied to formats not specified in the JSON. If you set it to ``"0"``, ingest will be disabled (but you can override this per-format). -Add a format name (DTA, POR, etc., as listed above) to change the limit for that particular format. - -Any size limits must be provided as string literals (in quotes), not number literals! +Add a format name (``csv``, ``dta``, etc., as listed above) to change the limit for that particular format. Examples: @@ -4429,6 +4430,9 @@ Examples: 2. If you want your Dataverse installation to not attempt to ingest XLSX files at all and apply a global limit of 512 MiB, use this setting: ``curl -X PUT -d '{"default":"536870912", "XSLX":"0"}' http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit`` +3. If you want your Dataverse installation to not attempt to ingest files at all except for CSV files that are 256 MiB or smaller, use this setting: + + ``curl -X PUT -d '{"default":"0", "CSV":"268435456"}' http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit`` :ZipUploadFilesLimit ++++++++++++++++++++ From 2450ecd0345377619de25f0990f564b8b52e23b5 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 8 Aug 2025 16:15:39 -0400 Subject: [PATCH 271/634] add more settings to enum to pass validation #11639 - :BagItHandlerEnabled - :BagValidatorJobPoolSize - :BagValidatorJobWaitInterval - :BagValidatorMaxErrors - :CreateDataFilesMaxErrorsToDisplay - :FileCategories - :LDNAnnounceRequiredFields - :LDNTarget These settings were originally added (not to the enum) in the following PRs: - https://github.com/IQSS/dataverse/pull/8478 - https://github.com/IQSS/dataverse/pull/8775 - https://github.com/IQSS/dataverse/pull/8677 --- doc/release-notes/11639-db-opts-idempotency.md | 8 ++++++++ .../iq/dataverse/DataFileCategoryServiceBean.java | 3 ++- .../harvard/iq/dataverse/EditDataFilesPageHelper.java | 3 ++- .../iq/dataverse/settings/SettingsServiceBean.java | 11 +++++++++++ .../harvard/iq/dataverse/util/bagit/BagValidator.java | 9 ++++++--- .../dataverse/util/file/BagItFileHandlerFactory.java | 3 ++- .../internalspi/LDNAnnounceDatasetVersionStep.java | 6 ++++-- 7 files changed, 35 insertions(+), 8 deletions(-) diff --git a/doc/release-notes/11639-db-opts-idempotency.md b/doc/release-notes/11639-db-opts-idempotency.md index 2a87a81eff8..55ed07c998c 100644 --- a/doc/release-notes/11639-db-opts-idempotency.md +++ b/doc/release-notes/11639-db-opts-idempotency.md @@ -25,11 +25,19 @@ Note: Despite the validation of setting names and languages, the content of any The following database settings are were added to the official list within the code (to remain valid with the settings cleanup mentioned above): - :BagGeneratorThreads +- :BagItHandlerEnabled - :BagItLocalPath +- :BagValidatorJobPoolSize +- :BagValidatorJobWaitInterval +- :BagValidatorMaxErrors - :BuiltinUsersKey - formerly BuiltinUsers.KEY +- :CreateDataFilesMaxErrorsToDisplay - :DRSArchiverConfig - a Harvard-specific setting - :DuraCloudContext - :DuraCloudHost - :DuraCloudPort +- :FileCategories - :GoogleCloudBucket - :GoogleCloudProject +- :LDNAnnounceRequiredFields +- :LDNTarget diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileCategoryServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileCategoryServiceBean.java index 29dcb22c3ec..d29b5670952 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileCategoryServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileCategoryServiceBean.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.FileCategories; import edu.harvard.iq.dataverse.util.BundleUtil; import jakarta.ejb.EJB; @@ -21,7 +22,7 @@ @Stateless public class DataFileCategoryServiceBean { - public static final String FILE_CATEGORIES_KEY = ":FileCategories"; + public static final String FILE_CATEGORIES_KEY = FileCategories.toString(); @EJB private SettingsServiceBean settingsService; diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDataFilesPageHelper.java b/src/main/java/edu/harvard/iq/dataverse/EditDataFilesPageHelper.java index 883baeedef4..7b5c3aa0857 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDataFilesPageHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDataFilesPageHelper.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.CreateDataFilesMaxErrorsToDisplay; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.file.CreateDataFileResult; import org.apache.commons.text.StringEscapeUtils; @@ -18,7 +19,7 @@ @Stateless public class EditDataFilesPageHelper { - public static final String MAX_ERRORS_TO_DISPLAY_SETTING = ":CreateDataFilesMaxErrorsToDisplay"; + public static final String MAX_ERRORS_TO_DISPLAY_SETTING = CreateDataFilesMaxErrorsToDisplay.toString(); public static final Integer MAX_ERRORS_TO_DISPLAY = 5; @Inject diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index b97de383391..7b6580068c0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -499,6 +499,13 @@ Whether Harvesting (OAI) service is enabled * :ArchiverSettings used by multiple archive commands. */ BagGeneratorThreads, + /** + * Various BagIt settings. + */ + BagValidatorJobPoolSize, + BagValidatorMaxErrors, + BagValidatorJobWaitInterval, + BagItHandlerEnabled, /** * A comma-separated list of roles for which new dataverses should inherit the * corresponding role assignments from the parent dataverse. Also affects @@ -675,6 +682,8 @@ Whether Harvesting (OAI) service is enabled * LDN Inbox Allowed Hosts - a comma separated list of IP addresses allowed to submit messages to the inbox */ LDNMessageHosts, + LDNAnnounceRequiredFields, + LDNTarget, /* * Allow a custom JavaScript to control values of specific fields. @@ -731,6 +740,8 @@ Whether Harvesting (OAI) service is enabled * files *with* the variable names line up top. */ StoreIngestedTabularFilesWithVarHeaders, + FileCategories, + CreateDataFilesMaxErrorsToDisplay, ContactFeedbackMessageSizeLimit, //Experimental setting to allow connecting to a GET external search service expecting a GET request with query parameter mirroring the search API query parameters (without search_service) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagValidator.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagValidator.java index a9052bf4c80..85a2f3f09ff 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagValidator.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagValidator.java @@ -1,5 +1,8 @@ package edu.harvard.iq.dataverse.util.bagit; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.BagValidatorJobPoolSize; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.BagValidatorJobWaitInterval; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.BagValidatorMaxErrors; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.bagit.BagValidation.FileValidationResult; import edu.harvard.iq.dataverse.util.bagit.ManifestReader.ManifestChecksum; @@ -27,9 +30,9 @@ public class BagValidator { private static final Logger logger = Logger.getLogger(BagValidator.class.getCanonicalName()); public static enum BagValidatorSettings { - JOB_POOL_SIZE(":BagValidatorJobPoolSize", 4), - MAX_ERRORS(":BagValidatorMaxErrors", 5), - JOB_WAIT_INTERVAL(":BagValidatorJobWaitInterval", 10); + JOB_POOL_SIZE(BagValidatorJobPoolSize.toString(), 4), + MAX_ERRORS(BagValidatorMaxErrors.toString(), 5), + JOB_WAIT_INTERVAL(BagValidatorJobWaitInterval.toString(), 10); private String settingsKey; private Integer defaultValue; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/file/BagItFileHandlerFactory.java b/src/main/java/edu/harvard/iq/dataverse/util/file/BagItFileHandlerFactory.java index 4b0263030dc..1d1b6b5b7aa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/file/BagItFileHandlerFactory.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/file/BagItFileHandlerFactory.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.util.file; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.BagItHandlerEnabled; import edu.harvard.iq.dataverse.util.bagit.BagValidator; import edu.harvard.iq.dataverse.util.bagit.BagValidator.BagValidatorSettings; import edu.harvard.iq.dataverse.util.bagit.ManifestReader; @@ -25,7 +26,7 @@ public class BagItFileHandlerFactory implements Serializable { private static final Logger logger = Logger.getLogger(BagItFileHandlerFactory.class.getCanonicalName()); - public static final String BAGIT_HANDLER_ENABLED_SETTING = ":BagItHandlerEnabled"; + public static final String BAGIT_HANDLER_ENABLED_SETTING = BagItHandlerEnabled.toString(); @EJB private SettingsServiceBean settingsService; diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java index 124eea801d9..d96c4db1305 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java @@ -5,6 +5,8 @@ import edu.harvard.iq.dataverse.DatasetFieldType; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.branding.BrandingUtil; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.LDNAnnounceRequiredFields; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.LDNTarget; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.bagit.OREMap; import edu.harvard.iq.dataverse.util.json.JsonLDTerm; @@ -52,8 +54,8 @@ public class LDNAnnounceDatasetVersionStep implements WorkflowStep { private static final Logger logger = Logger.getLogger(LDNAnnounceDatasetVersionStep.class.getName()); - private static final String REQUIRED_FIELDS = ":LDNAnnounceRequiredFields"; - private static final String LDN_TARGET = ":LDNTarget"; + private static final String REQUIRED_FIELDS = LDNAnnounceRequiredFields.toString(); + private static final String LDN_TARGET = LDNTarget.toString(); private static final String RELATED_PUBLICATION = "publication"; JsonLDTerm publicationIDType = null; From 692be46ed6f68d9e8a00e8af7028324bda4b3fec Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 8 Aug 2025 16:58:51 -0400 Subject: [PATCH 272/634] add TODOs for WorkflowsAdmin#IP_WHITELIST_KEY db setting #11639 --- .../harvard/iq/dataverse/api/Workflows.java | 1 + .../iq/dataverse/api/WorkflowsAdmin.java | 1 + .../edu/harvard/iq/dataverse/api/UtilIT.java | 19 ++++++ .../harvard/iq/dataverse/api/WorkflowsIT.java | 62 +++++++++++++++++++ tests/integration-tests.txt | 2 +- 5 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 src/test/java/edu/harvard/iq/dataverse/api/WorkflowsIT.java diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Workflows.java b/src/main/java/edu/harvard/iq/dataverse/api/Workflows.java index 4eadcedf71a..f4bac8e58cd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Workflows.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Workflows.java @@ -60,6 +60,7 @@ private boolean isAllowed(IpAddress addr) { private void updateWhitelist() { IpGroup updatedList = new IpGroup(); + // TODO: Investigate if this still works now that validateSettingName is in place. String[] ips = settingsSvc.get(WorkflowsAdmin.IP_WHITELIST_KEY, "127.0.0.1;::1").split(";"); Arrays.stream(ips) .forEach( str -> updatedList.add( diff --git a/src/main/java/edu/harvard/iq/dataverse/api/WorkflowsAdmin.java b/src/main/java/edu/harvard/iq/dataverse/api/WorkflowsAdmin.java index 15478aacff7..bf48ee660ad 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/WorkflowsAdmin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/WorkflowsAdmin.java @@ -31,6 +31,7 @@ @Path("admin/workflows") public class WorkflowsAdmin extends AbstractApiBean { + // TODO: To comply with validateSettingName, prepend with a colon? public static final String IP_WHITELIST_KEY="WorkflowsAdmin#IP_WHITELIST_KEY"; @EJB diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 548627554d2..b836d62492c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4641,6 +4641,25 @@ static Response updateDatasetTypeAvailableLicense(String idOrName, String jsonAr .put("/api/datasets/datasetTypes/" + idOrName + "/licenses"); } + public static Response getWorkflowIpWhitelist() { + Response response = given() + .get("/api/admin/workflows/ip-whitelist"); + return response; + } + + public static Response setWorkflowIpWhitelist(String iPWhitelist) { + Response response = given() + .body(iPWhitelist) + .put("/api/admin/workflows/ip-whitelist"); + return response; + } + + public static Response deleteWorkflowIpWhitelist() { + Response response = given() + .delete("/api/admin/workflows/ip-whitelist"); + return response; + } + static Response registerOidcUser(String jsonIn, String bearerToken) { return given() .header(HttpHeaders.AUTHORIZATION, bearerToken) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/WorkflowsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/WorkflowsIT.java new file mode 100644 index 00000000000..9e8c36b9e18 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/api/WorkflowsIT.java @@ -0,0 +1,62 @@ +package edu.harvard.iq.dataverse.api; + +import static edu.harvard.iq.dataverse.api.WorkflowsAdmin.IP_WHITELIST_KEY; +import io.restassured.RestAssured; +import static io.restassured.RestAssured.given; +import io.restassured.response.Response; +import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; +import static jakarta.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR; +import static jakarta.ws.rs.core.Response.Status.OK; +import static org.hamcrest.CoreMatchers.equalTo; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class WorkflowsIT { + + @BeforeAll + public static void setUpClass() { + RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); + + UtilIT.deleteWorkflowIpWhitelist(); + } + + @AfterAll + public static void afterClass() { + } + + @Test + public void testIpWhitelist() { + Response response = null; + + response = UtilIT.getWorkflowIpWhitelist(); + response.prettyPrint(); + response.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", equalTo("127.0.0.1;::1")); + + response = UtilIT.setWorkflowIpWhitelist("junk"); + response.prettyPrint(); + response.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo("Request contains illegal IP addresses.")); + + response = UtilIT.setWorkflowIpWhitelist("192.168.0.1;192.168.0.2"); + response.prettyPrint(); + response.then().assertThat() + .statusCode(OK.getStatusCode()); + + response = UtilIT.getWorkflowIpWhitelist(); + response.prettyPrint(); + response.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", equalTo("192.168.0.1;192.168.0.2")); + + // FIXME fix 500 error + response = given().when().get("/api/admin/settings/" + IP_WHITELIST_KEY); + response.prettyPrint(); + response.then().assertThat() + .statusCode(INTERNAL_SERVER_ERROR.getStatusCode()); + } + +} diff --git a/tests/integration-tests.txt b/tests/integration-tests.txt index 2a15ac3ce74..33d137c893a 100644 --- a/tests/integration-tests.txt +++ b/tests/integration-tests.txt @@ -1 +1 @@ -DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,HarvestingClientsIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT,MetadataBlocksIT,NetcdfIT,SignpostingIT,FitsIT,LogoutIT,DataRetrieverApiIT,ProvIT,S3AccessIT,OpenApiIT,InfoIT,DatasetFieldsIT,SavedSearchIT,DatasetTypesIT,DataverseFeaturedItemsIT,SendFeedbackApiIT,CustomizationIT,JsonLDExportIT +DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,HarvestingClientsIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT,MetadataBlocksIT,NetcdfIT,SignpostingIT,FitsIT,LogoutIT,DataRetrieverApiIT,ProvIT,S3AccessIT,OpenApiIT,InfoIT,DatasetFieldsIT,SavedSearchIT,DatasetTypesIT,DataverseFeaturedItemsIT,SendFeedbackApiIT,CustomizationIT,JsonLDExportIT,WorkflowsIT From a43da296892718fd38ded56ff6f4cfd976a0757b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 15 Sep 2025 14:15:56 +0200 Subject: [PATCH 273/634] style(settings): rename `WorkflowsAdmin#IP_WHITELIST_KEY` to `:WorkflowsAdminIpWhitelist` #11639 Updated `SettingsServiceBean`, `WorkflowsAdmin API`, and Flyway migration script to reflect the new name, enforcing alignment with the standardized naming pattern. Adjusted default value logic and separator constants. --- .../11639-db-opts-idempotency.md | 42 ++++++++++--------- .../harvard/iq/dataverse/api/Workflows.java | 5 ++- .../iq/dataverse/api/WorkflowsAdmin.java | 19 +++++---- .../settings/SettingsServiceBean.java | 5 +++ src/main/resources/db/migration/V6.8.0.1.sql | 11 ++++- .../harvard/iq/dataverse/api/WorkflowsIT.java | 16 +++---- 6 files changed, 59 insertions(+), 39 deletions(-) diff --git a/doc/release-notes/11639-db-opts-idempotency.md b/doc/release-notes/11639-db-opts-idempotency.md index 55ed07c998c..f73cbdebf83 100644 --- a/doc/release-notes/11639-db-opts-idempotency.md +++ b/doc/release-notes/11639-db-opts-idempotency.md @@ -5,9 +5,10 @@ With this release, we remove some legacy specialties around Database Settings an Most important changes: 1. Setting `BuiltinUsers.KEY` was renamed to `:BuiltinUsersKey`, aligned with our general naming pattern for options. -2. Setting `:TabularIngestSizeLimit` no longer uses suffixes for formats and becomes a JSON-based setting instead. -3. If set, both settings will be migrated to their new form automatically for you (Flyway migration). -4. You can no longer (accidentally) create or use arbitrary setting names or languages. +2. Setting `WorkflowsAdmin#IP_WHITELIST_KEY` was renamed to `:WorkflowsAdminIpWhitelist`, aligned with our general naming pattern for options. +3. Setting `:TabularIngestSizeLimit` no longer uses suffixes for formats and becomes a JSON-based setting instead. +4. If set, all three settings will be migrated to their new form automatically for you (Flyway migration). +5. You can no longer (accidentally) create or use arbitrary setting names or languages. All Admin API endpoints for settings now validate setting names and languages for existence and compliance. As an administrator of a Dataverse instance, you can now make use of enhanced Bulk Operations on the Settings Admin API: @@ -24,20 +25,21 @@ Note: Despite the validation of setting names and languages, the content of any The following database settings are were added to the official list within the code (to remain valid with the settings cleanup mentioned above): -- :BagGeneratorThreads -- :BagItHandlerEnabled -- :BagItLocalPath -- :BagValidatorJobPoolSize -- :BagValidatorJobWaitInterval -- :BagValidatorMaxErrors -- :BuiltinUsersKey - formerly BuiltinUsers.KEY -- :CreateDataFilesMaxErrorsToDisplay -- :DRSArchiverConfig - a Harvard-specific setting -- :DuraCloudContext -- :DuraCloudHost -- :DuraCloudPort -- :FileCategories -- :GoogleCloudBucket -- :GoogleCloudProject -- :LDNAnnounceRequiredFields -- :LDNTarget +- `:BagGeneratorThreads` +- `:BagItHandlerEnabled` +- `:BagItLocalPath` +- `:BagValidatorJobPoolSize` +- `:BagValidatorJobWaitInterval` +- `:BagValidatorMaxErrors` +- `:BuiltinUsersKey` - formerly `BuiltinUsers.KEY` +- `:CreateDataFilesMaxErrorsToDisplay` +- `:DRSArchiverConfig` - a Harvard-specific setting +- `:DuraCloudContext` +- `:DuraCloudHost` +- `:DuraCloudPort` +- `:FileCategories` +- `:GoogleCloudBucket` +- `:GoogleCloudProject` +- `:LDNAnnounceRequiredFields` +- `:LDNTarget` +- `:WorkflowsAdminIpWhitelist` - formerly `WorkflowsAdmin#IP_WHITELIST_KEY` diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Workflows.java b/src/main/java/edu/harvard/iq/dataverse/api/Workflows.java index f4bac8e58cd..7bd19b3a403 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Workflows.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Workflows.java @@ -3,6 +3,8 @@ import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.IpGroup; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddressRange; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key; import edu.harvard.iq.dataverse.workflow.PendingWorkflowInvocation; import edu.harvard.iq.dataverse.workflow.WorkflowServiceBean; import java.util.Arrays; @@ -60,8 +62,7 @@ private boolean isAllowed(IpAddress addr) { private void updateWhitelist() { IpGroup updatedList = new IpGroup(); - // TODO: Investigate if this still works now that validateSettingName is in place. - String[] ips = settingsSvc.get(WorkflowsAdmin.IP_WHITELIST_KEY, "127.0.0.1;::1").split(";"); + String[] ips = settingsSvc.getValueForKey(Key.WorkflowsAdminIpWhitelist, WorkflowsAdmin.DEFAULT_IP_ALLOWLIST).split(WorkflowsAdmin.IP_SEPARATOR); Arrays.stream(ips) .forEach( str -> updatedList.add( IpAddressRange.makeSingle( diff --git a/src/main/java/edu/harvard/iq/dataverse/api/WorkflowsAdmin.java b/src/main/java/edu/harvard/iq/dataverse/api/WorkflowsAdmin.java index bf48ee660ad..ecb7248cae9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/WorkflowsAdmin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/WorkflowsAdmin.java @@ -3,6 +3,8 @@ import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.util.json.JsonParseException; import edu.harvard.iq.dataverse.util.json.JsonParser; + +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.WorkflowsAdminIpWhitelist; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.brief; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.toJsonArray; @@ -30,9 +32,9 @@ */ @Path("admin/workflows") public class WorkflowsAdmin extends AbstractApiBean { - - // TODO: To comply with validateSettingName, prepend with a colon? - public static final String IP_WHITELIST_KEY="WorkflowsAdmin#IP_WHITELIST_KEY"; + + public static final String IP_SEPARATOR = ";"; + public static final String DEFAULT_IP_ALLOWLIST = "127.0.0.1" + IP_SEPARATOR + "::1"; @EJB WorkflowServiceBean workflows; @@ -154,14 +156,14 @@ public Response deleteWorkflow(@PathParam("id") String id ) { @Path("/ip-whitelist") @GET public Response getIpWhitelist() { - return ok( settingsSvc.get(IP_WHITELIST_KEY, "127.0.0.1;::1") ); + return ok( settingsSvc.getValueForKey(WorkflowsAdminIpWhitelist, DEFAULT_IP_ALLOWLIST) ); } @Path("/ip-whitelist") @PUT public Response setIpWhitelist(String body) { String ipList = body.trim(); - String[] ips = ipList.split(";"); + String[] ips = ipList.split(IP_SEPARATOR); boolean allIpsOk = Arrays.stream(ips).allMatch(ip->{ try { IpAddress.valueOf(ip); @@ -171,18 +173,17 @@ public Response setIpWhitelist(String body) { } } ); if (allIpsOk) { - settingsSvc.set(IP_WHITELIST_KEY, ipList); - return ok( settingsSvc.get(IP_WHITELIST_KEY, "127.0.0.1;::1") ); + settingsSvc.setValueForKey(WorkflowsAdminIpWhitelist, ipList); + return ok( settingsSvc.getValueForKey(WorkflowsAdminIpWhitelist, DEFAULT_IP_ALLOWLIST) ); } else { return badRequest("Request contains illegal IP addresses."); } - } @Path("/ip-whitelist") @DELETE public Response deleteIpWhitelist() { - settingsSvc.delete(IP_WHITELIST_KEY); + settingsSvc.deleteValueForKey(WorkflowsAdminIpWhitelist); return ok( "Restored whitelist to default (127.0.0.1;::1)" ); } diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 7b6580068c0..e23727ed7eb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -171,6 +171,11 @@ public enum Key { @Deprecated(forRemoval = true, since = "2025-04-29") BlockedApiPolicy, + /** + * Semicolon separated allowlist of IP addresses allowed administrative access to workflows. + */ + WorkflowsAdminIpWhitelist, + /** * A special secret that, if set, needs to be given when trying to manage internal users. * This key was formerly known as "BuiltinUsers.KEY", which never was a setting name aligning with the others. diff --git a/src/main/resources/db/migration/V6.8.0.1.sql b/src/main/resources/db/migration/V6.8.0.1.sql index 656ef04d572..8e810270b06 100644 --- a/src/main/resources/db/migration/V6.8.0.1.sql +++ b/src/main/resources/db/migration/V6.8.0.1.sql @@ -85,4 +85,13 @@ DO $$ INSERT INTO setting (name, lang, content) VALUES (':BuiltinUsersKey', NULL, (SELECT content FROM setting WHERE name = 'BuiltinUsers.KEY')); DELETE FROM setting WHERE name = 'BuiltinUsers.KEY'; END IF; - END $$; \ No newline at end of file + END $$; + +-- 3. Migrate WorkflowsAdmin#IP_WHITELIST_KEY to the new setting name +DO $$ + BEGIN + IF EXISTS (SELECT 1 FROM setting WHERE name = 'WorkflowsAdmin#IP_WHITELIST_KEY') THEN + INSERT INTO setting (name, lang, content) VALUES (':WorkflowsAdminIpWhitelist', NULL, (SELECT content FROM setting WHERE name = 'WorkflowsAdmin#IP_WHITELIST_KEY')); + DELETE FROM setting WHERE name = 'WorkflowsAdmin#IP_WHITELIST_KEY'; + END IF; + END $$; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/WorkflowsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/WorkflowsIT.java index 9e8c36b9e18..4b94fe6ee68 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/WorkflowsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/WorkflowsIT.java @@ -1,6 +1,6 @@ package edu.harvard.iq.dataverse.api; -import static edu.harvard.iq.dataverse.api.WorkflowsAdmin.IP_WHITELIST_KEY; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import io.restassured.RestAssured; import static io.restassured.RestAssured.given; import io.restassured.response.Response; @@ -35,13 +35,15 @@ public void testIpWhitelist() { .statusCode(OK.getStatusCode()) .body("data.message", equalTo("127.0.0.1;::1")); + String testIp = "192.168.0.1;192.168.0.2"; + response = UtilIT.setWorkflowIpWhitelist("junk"); response.prettyPrint(); response.then().assertThat() .statusCode(BAD_REQUEST.getStatusCode()) .body("message", equalTo("Request contains illegal IP addresses.")); - response = UtilIT.setWorkflowIpWhitelist("192.168.0.1;192.168.0.2"); + response = UtilIT.setWorkflowIpWhitelist(testIp); response.prettyPrint(); response.then().assertThat() .statusCode(OK.getStatusCode()); @@ -50,13 +52,13 @@ public void testIpWhitelist() { response.prettyPrint(); response.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data.message", equalTo("192.168.0.1;192.168.0.2")); - - // FIXME fix 500 error - response = given().when().get("/api/admin/settings/" + IP_WHITELIST_KEY); + .body("data.message", equalTo(testIp)); + + response = given().when().get("/api/admin/settings/" + SettingsServiceBean.Key.WorkflowsAdminIpWhitelist); response.prettyPrint(); response.then().assertThat() - .statusCode(INTERNAL_SERVER_ERROR.getStatusCode()); + .statusCode(OK.getStatusCode()) + .body("data.message", equalTo(testIp)); } } From 2a97dd74e043dce8fc89544aabddecc64a03e5ad Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 15 Sep 2025 14:22:46 +0200 Subject: [PATCH 274/634] chore(settings): mark `SettingsServiceBean` methods using String for lookup as deprecated for removal In the future, we should entirely remove lookup of arbitrary strings in the database for settings. Instead, we need a discovery mechanism for keys that allow plugins to register and lookup their own database settings. --- .../iq/dataverse/settings/SettingsServiceBean.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index e23727ed7eb..e5fce1a2349 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -817,6 +817,7 @@ public static SettingsServiceBean.Key parse(String key) { * @param name of the setting * @return the actual setting, or {@code null}. */ + @Deprecated(since = "6.9", forRemoval = true) public String get( String name ) { List tokens = em.createNamedQuery("Setting.findByName", Setting.class) .setParameter("name", name ) @@ -961,11 +962,13 @@ public Boolean getValueForCompoundKeyAsBoolean(Key key, String param) { * @param defaultValue The value to return if no setting is found in the DB. * @return Either the stored value, or the default value. */ + @Deprecated(since = "6.9", forRemoval = true) public String get( String name, String defaultValue ) { String val = get(name); return (val!=null) ? val : defaultValue; } + @Deprecated(since = "6.9", forRemoval = true) public String get(String name, String lang, String defaultValue ) { // Database safeguard, as the default is an empty string if (lang == null) lang = ""; @@ -991,7 +994,8 @@ public String getValueForKey( Key key, String lang, String defaultValue ) { return get( key.toString(), lang, defaultValue ); } - + + @Deprecated(since = "6.9", forRemoval = true) public Setting set( String name, String content ) { Setting s = null; @@ -1015,6 +1019,7 @@ public Setting set( String name, String content ) { return s; } + @Deprecated(since = "6.9", forRemoval = true) public Setting set( String name, String lang, String content ) { // Database safeguard, as the default is an empty string if (lang == null) lang = ""; @@ -1053,6 +1058,7 @@ public Setting setValueForKey( Key key, String content ) { * @param defaultValue logical value of {@code null}. * @return boolean value of the setting. */ + @Deprecated(since = "6.9", forRemoval = true) public boolean isTrue( String name, boolean defaultValue ) { String val = get(name); return ( val==null ) ? defaultValue : StringUtil.isTrue(val); @@ -1079,6 +1085,7 @@ public void deleteValueForKey( Key name ) { delete( name.toString() ); } + @Deprecated(since = "6.9", forRemoval = true) public void delete( String name ) { actionLogSvc.log( new ActionLogRecord(ActionLogRecord.ActionType.Setting, "delete") .setInfo(name)); @@ -1087,6 +1094,7 @@ public void delete( String name ) { .executeUpdate(); } + @Deprecated(since = "6.9", forRemoval = true) public void delete( String name, String lang ) { // Database safeguard, as the default is an empty string if (lang == null) lang = ""; From c667b18efa01e44b11abb406b108805d859a547f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 15 Sep 2025 14:22:58 +0200 Subject: [PATCH 275/634] doc(settings): clarify database changes require app reload for ORM --- doc/sphinx-guides/source/installation/config.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index f33c04eada8..c3f97904ead 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3873,7 +3873,8 @@ You might also create your own profiles and use these, please refer to the upstr Database Settings ----------------- -These settings are stored in the ``setting`` database table but we recommend using the Admin API (:ref:`admin-api-db-settings`) to view and modify them, as shown below. +These settings are stored in the ``setting`` database table but we recommend using the Settings Admin API (:ref:`admin-api-db-settings`) to view and modify them, as shown below. +If changed in the database directly, you need to reload the application to make the ORM pickup the changes. In short: From b3a4c4f76aff872737dcfada9e6cbd31153b294b Mon Sep 17 00:00:00 2001 From: Steven Ferey Date: Wed, 17 Sep 2025 15:59:03 +0200 Subject: [PATCH 276/634] allows carriage return of long strings in dataset metadata --- src/main/webapp/resources/css/structure.css | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index 4d83ddd489d..8d856f74f71 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -600,6 +600,11 @@ div[id$='roleDisplay'] span.label, div[id$='roleDetails'] span.label {display:in #title {font-size:36px !important;line-height:1.1;font-weight:bold;margin-top:0px;margin-bottom:0px;} #author, #contact {display:block; margin-bottom:.5em;} +table.metadata { + word-break: normal; + overflow-wrap: anywhere; +} + /* #11053 */ #publication span { margin: 0px 5px 0px 5px; From 991ed05d4fa35ebcec1f5ff69f91d3081ccbde7f Mon Sep 17 00:00:00 2001 From: Steven Ferey Date: Wed, 17 Sep 2025 16:01:34 +0200 Subject: [PATCH 277/634] add doc --- src/main/webapp/resources/css/structure.css | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index 8d856f74f71..ac42ff44f6b 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -600,6 +600,7 @@ div[id$='roleDisplay'] span.label, div[id$='roleDetails'] span.label {display:in #title {font-size:36px !important;line-height:1.1;font-weight:bold;margin-top:0px;margin-bottom:0px;} #author, #contact {display:block; margin-bottom:.5em;} +/* #11824 */ table.metadata { word-break: normal; overflow-wrap: anywhere; From 2aaf139bcfb4f87fe4f4b1ff4d5b7bfedff15bbd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Sep 2025 20:25:10 +0000 Subject: [PATCH 278/634] Bump org.keycloak:keycloak-services in /conf/keycloak/builtin-users-spi Bumps [org.keycloak:keycloak-services](https://github.com/keycloak/keycloak) from 26.3.2 to 26.3.3. - [Release notes](https://github.com/keycloak/keycloak/releases) - [Commits](https://github.com/keycloak/keycloak/compare/26.3.2...26.3.3) --- updated-dependencies: - dependency-name: org.keycloak:keycloak-services dependency-version: 26.3.3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- conf/keycloak/builtin-users-spi/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/keycloak/builtin-users-spi/pom.xml b/conf/keycloak/builtin-users-spi/pom.xml index 36cf6548d01..8a4c6c5a5ad 100644 --- a/conf/keycloak/builtin-users-spi/pom.xml +++ b/conf/keycloak/builtin-users-spi/pom.xml @@ -100,7 +100,7 @@ - 26.3.2 + 26.3.3 17 3.2.0 0.4 From ff2e890736ff6735e50912b2c539c23bca110bed Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 17 Sep 2025 17:00:20 -0400 Subject: [PATCH 279/634] Payara 6.2025.9 update --- .github/workflows/container_maintenance.yml | 4 ++-- doc/sphinx-guides/source/container/base-image.rst | 2 +- doc/sphinx-guides/source/developers/classic-dev-env.rst | 6 +++--- doc/sphinx-guides/source/installation/prerequisites.rst | 6 +++--- doc/sphinx-guides/source/qa/test-automation.md | 2 +- modules/dataverse-parent/pom.xml | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 142363cbe1a..744de1c5165 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -63,11 +63,11 @@ jobs: - discover outputs: # This is a JSON map with keys of branch names (supported releases & develop) and values containing an array of known image tags for the branch - # Example: {"v6.6": ["latest", "6.6-noble", "6.6-noble-r1"], "v6.5": ["6.5-noble", "6.5-noble-r5"], "v6.4": ["6.4-noble", "6.4-noble-r12"], "develop": ["unstable", "6.7-noble", "6.7-noble-p6.2025.3-j17"]} + # Example: {"v6.6": ["latest", "6.6-noble", "6.6-noble-r1"], "v6.5": ["6.5-noble", "6.5-noble-r5"], "v6.4": ["6.4-noble", "6.4-noble-r12"], "develop": ["unstable", "6.7-noble", "6.7-noble-p6.2025.9-j17"]} supported_tag_matrix: ${{ steps.execute.outputs.supported_tag_matrix }} # This is a JSON list containing a flattened map of branch names and the latest non-rolling tag - # Example: [ "v6.6=gdcc/base:6.6-noble-r1", "v6.5=gdcc/base:6.5-noble-r5", "v6.4=gdcc/base:6.4-noble-r12", "develop=gdcc/base:6.7-noble-p6.2025.3-j17" ] + # Example: [ "v6.6=gdcc/base:6.6-noble-r1", "v6.5=gdcc/base:6.5-noble-r5", "v6.4=gdcc/base:6.4-noble-r12", "develop=gdcc/base:6.7-noble-p6.2025.9-j17" ] rebuilt_images: ${{ steps.execute.outputs.rebuilt_images }} steps: diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index 4adc6bb6fb1..0720995deb2 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -82,7 +82,7 @@ For now, stale images will be kept on Docker Hub indefinitely. Will roll over to the rolling production tag after a Dataverse release. - | **Flexible Stack** | Definition: ``.--p-j`` - | Example: :substitution-code:`|nextVersion|-noble-p6.2025.3-j17` + | Example: :substitution-code:`|nextVersion|-noble-p6.2025.9-j17` | Summary: Rolling tag during a development cycle of the Dataverse software (`Dockerfile `__). **NOTE**: In these tags for development usage, the version number will always be 1 minor version ahead of existing Dataverse releases. diff --git a/doc/sphinx-guides/source/developers/classic-dev-env.rst b/doc/sphinx-guides/source/developers/classic-dev-env.rst index d1f54fd9d5f..1c9e4d5c35d 100755 --- a/doc/sphinx-guides/source/developers/classic-dev-env.rst +++ b/doc/sphinx-guides/source/developers/classic-dev-env.rst @@ -93,15 +93,15 @@ On Linux, install ``jq`` from your package manager or download a binary from htt Install Payara ~~~~~~~~~~~~~~ -Payara 6.2025.3 or higher is required. +Payara 6.2025.9 or higher is required. To install Payara, run the following commands: ``cd /usr/local`` -``sudo curl -O -L https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2025.3/payara-6.2025.3.zip`` +``sudo curl -O -L https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2025.9/payara-6.2025.9.zip`` -``sudo unzip payara-6.2025.3.zip`` +``sudo unzip payara-6.2025.9.zip`` ``sudo chown -R $USER /usr/local/payara6`` diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index a5aacc4701c..2b456f7c0ed 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -44,7 +44,7 @@ On RHEL/derivative you can make Java 17 the default with the ``alternatives`` co Payara ------ -Payara 6.2025.3 is recommended. Newer versions might work fine. Regular updates are recommended. +Payara 6.2025.9 is recommended. Newer versions might work fine. Regular updates are recommended. Installing Payara ================= @@ -55,8 +55,8 @@ Installing Payara - Download and install Payara (installed in ``/usr/local/payara6`` in the example commands below):: - # wget https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2025.3/payara-6.2025.3.zip - # unzip payara-6.2025.3.zip + # wget https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2025.9/payara-6.2025.9.zip + # unzip payara-6.2025.9.zip # mv payara6 /usr/local If nexus.payara.fish is ever down for maintenance, Payara distributions are also available from https://repo1.maven.org/maven2/fish/payara/distributions/payara/ diff --git a/doc/sphinx-guides/source/qa/test-automation.md b/doc/sphinx-guides/source/qa/test-automation.md index fa995bcaafd..7650b41db81 100644 --- a/doc/sphinx-guides/source/qa/test-automation.md +++ b/doc/sphinx-guides/source/qa/test-automation.md @@ -52,7 +52,7 @@ Go to the end of the log and then scroll up, looking for the failure. A failed A ``` TASK [dataverse : download payara zip] ***************************************** -fatal: [localhost]: FAILED! => {"changed": false, "dest": "/tmp/payara.zip", "elapsed": 10, "msg": "Request failed: ", "url": "https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2025.3/payara-6.2025.3.zip"} +fatal: [localhost]: FAILED! => {"changed": false, "dest": "/tmp/payara.zip", "elapsed": 10, "msg": "Request failed: ", "url": "https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2025.9/payara-6.2025.9.zip"} ``` In the example above, if Payara can't be downloaded, we're obviously going to have problems deploying Dataverse to it! diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 0cb83fffc0f..11e761e2a77 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -149,7 +149,7 @@ -Duser.timezone=${project.timezone} -Dfile.encoding=${project.build.sourceEncoding} -Duser.language=${project.language} -Duser.region=${project.region} - 6.2025.3 + 6.2025.9 42.7.7 9.8.0 2.33.0 From bc330b73a38166359697eaf1a2abf3bb3d9c7ff4 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 17 Sep 2025 17:04:22 -0400 Subject: [PATCH 280/634] release note --- doc/release-notes/11827-Payara-6.2025.9.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/release-notes/11827-Payara-6.2025.9.md diff --git a/doc/release-notes/11827-Payara-6.2025.9.md b/doc/release-notes/11827-Payara-6.2025.9.md new file mode 100644 index 00000000000..1a084312358 --- /dev/null +++ b/doc/release-notes/11827-Payara-6.2025.9.md @@ -0,0 +1,3 @@ +The recommended Payara version has been updated to Payara-6.2025.9 + +Standard Payara upgrade instructions - as in the 6.7 release notes should be added. From 5c8a4eb685bd4927572763da157b161ecc4ec520 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 18 Sep 2025 15:43:24 -0400 Subject: [PATCH 281/634] #11710 add already linking functionality --- .../dataverse/DatasetLinkingServiceBean.java | 1 - .../harvard/iq/dataverse/api/Dataverses.java | 5 ++- .../impl/GetLinkingDataverseListCommand.java | 43 +++++++++++++++---- .../iq/dataverse/api/DataversesIT.java | 8 ++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 14 +++++- 5 files changed, 58 insertions(+), 13 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetLinkingServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetLinkingServiceBean.java index 39c82bfa3f1..514ecb1dc4d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetLinkingServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetLinkingServiceBean.java @@ -13,7 +13,6 @@ import jakarta.persistence.EntityManager; import jakarta.persistence.NoResultException; import jakarta.persistence.PersistenceContext; -import jakarta.persistence.Query; import jakarta.persistence.TypedQuery; /** diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index f5816c36852..7322a8c9341 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1777,7 +1777,7 @@ public Response linkDataverse(@Context ContainerRequestContext crc, @PathParam(" @AuthRequired @Produces(MediaType.APPLICATION_JSON) @Path("{identifier}/{type}/linkingDataverses") - public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @QueryParam("searchTerm") String searchTerm, @PathParam("type") String type) { + public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @QueryParam("searchTerm") String searchTerm, @QueryParam("alreadyLinking") boolean alreadyLinking, @PathParam("type") String type) { try { @@ -1785,7 +1785,8 @@ public Response getLinkingDataverseList(@Context ContainerRequestContext crc, @P List dataversesForLinking = execCommand(new GetLinkingDataverseListCommand( createDataverseRequest(getRequestUser(crc)), dvObject, - searchTerm + searchTerm, + alreadyLinking )); JsonArrayBuilder dvBuilder = Json.createArrayBuilder(); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java index c47c6215640..de8e5a62938 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java @@ -25,13 +25,31 @@ public class GetLinkingDataverseListCommand extends AbstractCommand execute(CommandContext ctxt) throws CommandException { @@ -57,14 +75,21 @@ public List execute(CommandContext ctxt) throws CommandException { } else { permToCheck = Permission.LinkDataverse; } - - dataversesForLinking = ctxt.permissions().findPermittedCollections(getRequest(), authUser, permToCheck, searchParam); - //Don't bother with checking for already linked if there are none to be tested. - if(dataversesForLinking == null || dataversesForLinking.isEmpty()) { - return dataversesForLinking; + //dependin on the already linked boolean the command will return a list of Dataverses available for linking + // or a list of dataverses to which the object has already been linked - for the unlink function + if (!alreadyLinked) { + dataversesForLinking = ctxt.permissions().findPermittedCollections(getRequest(), authUser, permToCheck, searchParam); + //Don't bother with checking for already linked if there are none to be tested. + if (dataversesForLinking == null || dataversesForLinking.isEmpty()) { + return dataversesForLinking; + } + return ctxt.dataverses().removeUnlinkableDataverses(dataversesForLinking, dvObject); + } else { + if (dvObject instanceof Dataset) {; + return ctxt.dsLinking().findLinkingDataverses(dvObject.getId()); + } else { + return ctxt.dvLinking().findLinkingDataverses(dvObject.getId()); + } } - return ctxt.dataverses().removeUnlinkableDataverses(dataversesForLinking, dvObject); - } - } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index bbe1291f90f..86b082d79ae 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -836,6 +836,14 @@ public void testGetLinkableDataverses(){ .statusCode(OK.getStatusCode()) .body("data.size()", equalTo(0)); + //if you ask for already linked you should get one + getLinkableDataverses = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiToken, dataverseAliasForLinking, true); + + getLinkableDataverses.prettyPrint(); + getLinkableDataverses.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", equalTo(1)); + //set user api back to super user for cleanup UtilIT.setSuperuserStatus(username, Boolean.TRUE); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 5f5416e08cc..47b8deaf0e4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4730,19 +4730,31 @@ static Response updateDataverseFeaturedItem(long featuredItemId, } static Response getLinkableDataverses(String type, String dvObjectId, String apiToken, String searchTerm) { + return getLinkableDataverses(type, dvObjectId, apiToken, searchTerm, false); + } + + static Response getLinkableDataverses(String type, String dvObjectId, String apiToken, String searchTerm, boolean alreadyLinked) { String idInPath = dvObjectId; // Assume it's a number to start. String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. if (type.equals("dataset")) { if (!NumberUtils.isCreatable(idInPath)) { idInPath = ":persistentId"; - if (searchTerm == null) { + if (searchTerm == null || searchTerm.isEmpty() ) { optionalQueryParam = "?persistentId=" + dvObjectId; } else { optionalQueryParam = "&persistentId=" + dvObjectId; } } } + + if (alreadyLinked){ + if (optionalQueryParam.isEmpty()){ + optionalQueryParam = "?alreadyLinking=true"; + } else { + optionalQueryParam = optionalQueryParam + "&alreadyLinking=true"; + } + } if (searchTerm == null) { if (!apiToken.isEmpty()) { From eb4ed35865a9cd7d7555805d4a7da8eb21818090 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 18 Sep 2025 16:41:47 -0400 Subject: [PATCH 282/634] #11710 typo --- .../engine/command/impl/GetLinkingDataverseListCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java index de8e5a62938..793175d26b7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java @@ -85,7 +85,7 @@ public List execute(CommandContext ctxt) throws CommandException { } return ctxt.dataverses().removeUnlinkableDataverses(dataversesForLinking, dvObject); } else { - if (dvObject instanceof Dataset) {; + if (dvObject instanceof Dataset) { return ctxt.dsLinking().findLinkingDataverses(dvObject.getId()); } else { return ctxt.dvLinking().findLinkingDataverses(dvObject.getId()); From 1deda57d6d85eda2003c26fff93f3cca6845c713 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 18 Jul 2025 11:38:01 -0400 Subject: [PATCH 283/634] add retries to DataCite calls for 429, 503, 504 --- .../doi/datacite/DataCiteRESTfullClient.java | 93 ++++++++++++++++--- 1 file changed, 82 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteRESTfullClient.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteRESTfullClient.java index 465b10ee407..47394f0ad54 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteRESTfullClient.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteRESTfullClient.java @@ -41,6 +41,10 @@ public class DataCiteRESTfullClient implements Closeable { private static final Logger logger = Logger.getLogger(DataCiteRESTfullClient.class.getCanonicalName()); + // Constants for retry mechanism + private static final int MAX_RETRIES = 5; + private static final long RETRY_DELAY_MS = 10000; // 10 seconds + private String url; private CloseableHttpClient httpClient; private HttpClientContext context; @@ -59,11 +63,78 @@ public DataCiteRESTfullClient(String url, String username, String password) { public void close() { if (this.httpClient != null) { try { - httpClient.close(); + httpClient.close(); } catch (IOException io) { - logger.warning("IOException closing hhtpClient: " + io.getMessage()); - } + logger.warning("IOException closing httpClient: " + io.getMessage()); + } + } + } + + /** + * Execute HTTP request with retry mechanism for specific status codes + * + * @param request The HTTP request to execute + * @param operationName Name of the operation for logging + * @return HttpResponse The response from the server + * @throws IOException If an error occurs during the request + */ + private HttpResponse executeWithRetry(org.apache.http.client.methods.HttpRequestBase request, String operationName) throws IOException { + int attempts = 0; + IOException lastException = null; + + while (attempts < MAX_RETRIES) { + try { + HttpResponse response = httpClient.execute(request, context); + int statusCode = response.getStatusLine().getStatusCode(); + + // If we get a retry status code, try again after delay + if (statusCode == 429 || statusCode == 503 || statusCode == 504) { + EntityUtils.consumeQuietly(response.getEntity()); + attempts++; + + if (attempts < MAX_RETRIES) { + logger.warning("DataCite API returned status " + statusCode + + " for " + operationName + ". Retrying in " + + (RETRY_DELAY_MS / 1000) + " seconds (attempt " + attempts + " of " + MAX_RETRIES + ")"); + try { + Thread.sleep(RETRY_DELAY_MS); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IOException("Retry interrupted", ie); + } + } else { + logger.severe("DataCite API failed with status " + statusCode + + " for " + operationName + " after " + MAX_RETRIES + " attempts"); + return response; // Return the last failed response + } + } else { + // Success or non-retry error code + return response; + } + } catch (IOException ioe) { + lastException = ioe; + attempts++; + + if (attempts < MAX_RETRIES) { + logger.warning("IOException during " + operationName + ": " + ioe.getMessage() + + ". Retrying in " + (RETRY_DELAY_MS / 1000) + " seconds (attempt " + + attempts + " of " + MAX_RETRIES + ")"); + try { + Thread.sleep(RETRY_DELAY_MS); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IOException("Retry interrupted", ie); + } + } else { + logger.severe("DataCite API failed for " + operationName + " after " + + MAX_RETRIES + " attempts due to: " + ioe.getMessage()); + throw lastException; + } + } } + + // This should never happen, but just in case + throw new IOException("Failed to execute request after " + MAX_RETRIES + " attempts"); } /** @@ -75,7 +146,7 @@ public void close() { public String getUrl(String doi) { HttpGet httpGet = new HttpGet(this.url + "/doi/" + doi); try { - HttpResponse response = httpClient.execute(httpGet,context); + HttpResponse response = executeWithRetry(httpGet, "getUrl"); HttpEntity entity = response.getEntity(); String data = null; @@ -104,7 +175,7 @@ public String postUrl(String doi, String url) throws IOException { httpPost.setHeader("Content-Type", "text/plain;charset=UTF-8"); httpPost.setEntity(new StringEntity("doi=" + doi + "\nurl=" + url, "utf-8")); - HttpResponse response = httpClient.execute(httpPost, context); + HttpResponse response = executeWithRetry(httpPost, "postUrl"); String data = EntityUtils.toString(response.getEntity(), encoding); if (response.getStatusLine().getStatusCode() != 201) { String errMsg = "Response from postUrl: " + response.getStatusLine().getStatusCode() + ", " + data; @@ -124,7 +195,7 @@ public String getMetadata(String doi) { HttpGet httpGet = new HttpGet(this.url + "/metadata/" + doi); httpGet.setHeader("Accept", "application/xml"); try { - HttpResponse response = httpClient.execute(httpGet,context); + HttpResponse response = executeWithRetry(httpGet, "getMetadata"); String data = EntityUtils.toString(response.getEntity(), encoding); if (response.getStatusLine().getStatusCode() != 200) { String errMsg = "Response from getMetadata: " + response.getStatusLine().getStatusCode() + ", " + data; @@ -133,7 +204,7 @@ public String getMetadata(String doi) { } return data; } catch (IOException ioe) { - logger.log(Level.SEVERE, "IOException when get metadata"); + logger.log(Level.SEVERE, "IOException when get metadata", ioe); throw new RuntimeException("IOException when get metadata", ioe); } } @@ -147,7 +218,7 @@ public String getMetadata(String doi) { public boolean testDOIExists(String doi) throws IOException { HttpGet httpGet = new HttpGet(this.url + "/metadata/" + doi); httpGet.setHeader("Accept", "application/xml"); - HttpResponse response = httpClient.execute(httpGet, context); + HttpResponse response = executeWithRetry(httpGet, "testDOIExists"); if (response.getStatusLine().getStatusCode() != 200) { EntityUtils.consumeQuietly(response.getEntity()); return false; @@ -166,7 +237,7 @@ public String postMetadata(String metadata) throws IOException { HttpPost httpPost = new HttpPost(this.url + "/metadata"); httpPost.setHeader("Content-Type", "application/xml;charset=UTF-8"); httpPost.setEntity(new StringEntity(metadata, "utf-8")); - HttpResponse response = httpClient.execute(httpPost, context); + HttpResponse response = executeWithRetry(httpPost, "postMetadata"); String data = EntityUtils.toString(response.getEntity(), encoding); if (response.getStatusLine().getStatusCode() != 201) { String errMsg = "Response from postMetadata: " + response.getStatusLine().getStatusCode() + ", " + data; @@ -185,7 +256,7 @@ public String postMetadata(String metadata) throws IOException { public String inactiveDataset(String doi) { HttpDelete httpDelete = new HttpDelete(this.url + "/metadata/" + doi); try { - HttpResponse response = httpClient.execute(httpDelete,context); + HttpResponse response = executeWithRetry(httpDelete, "inactiveDataset"); String data = EntityUtils.toString(response.getEntity(), encoding); if (response.getStatusLine().getStatusCode() != 200) { String errMsg = "Response code: " + response.getStatusLine().getStatusCode() + ", " + data; @@ -194,7 +265,7 @@ public String inactiveDataset(String doi) { } return data; } catch (IOException ioe) { - logger.log(Level.SEVERE, "IOException when inactive dataset"); + logger.log(Level.SEVERE, "IOException when inactive dataset", ioe); throw new RuntimeException("IOException when inactive dataset", ioe); } } From e16b29ca75160cc96f8e2742b69b208c03e9ed6c Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 18 Jul 2025 12:38:36 -0400 Subject: [PATCH 284/634] optionally update DataCite DOI when needed during publish --- .../pidproviders/doi/datacite/DataCiteDOIProvider.java | 7 ++++++- .../edu/harvard/iq/dataverse/settings/FeatureFlags.java | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java index dd64a89dfe6..3c21699d45e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java @@ -15,6 +15,7 @@ import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; +import edu.harvard.iq.dataverse.settings.FeatureFlags; import edu.harvard.iq.dataverse.util.json.JsonUtil; import jakarta.json.JsonObject; @@ -217,7 +218,11 @@ public boolean publicizeIdentifier(DvObject dvObject) { metadata.put("datacite.publicationyear", generateYear(dvObject)); metadata.put("_target", getTargetUrl(dvObject)); try { - doiDataCiteRegisterService.registerIdentifier(identifier, metadata, dvObject); + if (FeatureFlags.ONLY_UPDATE_DATACITE_WHEN_NEEDED.enabled()) { + doiDataCiteRegisterService.reRegisterIdentifier(identifier, metadata, dvObject); + } else { + doiDataCiteRegisterService.registerIdentifier(identifier, metadata, dvObject); + } return true; } catch (Exception e) { logger.log(Level.WARNING, "modifyMetadata failed: " + e.getMessage(), e); diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java index ae117159b5c..0e4bb510f4b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java @@ -229,7 +229,12 @@ public enum FeatureFlags { * @since Dataverse 6.8 */ ENABLE_PID_FAILURE_LOG("enable-pid-failure-log"), - + + /** + * Only update a DataCite DOI when needed (for efficiency, lighter load on DataCite). + */ + ONLY_UPDATE_DATACITE_WHEN_NEEDED("only-update-datacite-when-needed"), + ; final String flag; From 6c086e33479d60c53199ce5ccf3f651353ca84d7 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 18 Jul 2025 13:02:42 -0400 Subject: [PATCH 285/634] catch null url for draft DOI --- .../doi/datacite/DOIDataCiteRegisterService.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DOIDataCiteRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DOIDataCiteRegisterService.java index a4d788de4df..d9ddfe04393 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DOIDataCiteRegisterService.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DOIDataCiteRegisterService.java @@ -95,7 +95,14 @@ public String reRegisterIdentifier(String identifier, Map metada } retString = "metadata:\\r" + client.postMetadata(xmlMetadata) + "\\r"; } - if (!target.equals(client.getUrl(numericIdentifier))) { + String currentUrl = null; + try { + //May get a 204 if the DOI is still draft + currentUrl = client.getUrl(numericIdentifier); + } catch (RuntimeException ex) { + logger.fine("Error getting Url for " + numericIdentifier + ": " + ex.getMessage()); + } + if (!target.equals(currentUrl)) { logger.info("Updating target URL to " + target); client.postUrl(numericIdentifier, target); retString = retString + "url:\\r" + target; From eda19fe2a08571465ac039f5398d0c5887249869 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 19 Sep 2025 11:54:52 -0400 Subject: [PATCH 286/634] docs, release note --- doc/release-notes/11832-DataCite-scaling.md | 14 ++++++++++++++ doc/sphinx-guides/source/installation/config.rst | 10 ++++++++++ .../iq/dataverse/settings/FeatureFlags.java | 11 ++++++++++- 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 doc/release-notes/11832-DataCite-scaling.md diff --git a/doc/release-notes/11832-DataCite-scaling.md b/doc/release-notes/11832-DataCite-scaling.md new file mode 100644 index 00000000000..574ef05f94c --- /dev/null +++ b/doc/release-notes/11832-DataCite-scaling.md @@ -0,0 +1,14 @@ +This release adds functionality to retry calls to DataCite when their server is overloaded or Dataverse has hit their rate limit. + +It also introduces an option to only update DataCite metadata after checking to see if the current DataCite information is out of date. +(This adds a request to get information from DataCite before any potential write of new information which will be more efficient when +most DOIs have not changed but will result in an extra call to get info when a DOI has changed.) + +Both of these can help when DataCite is being used heavily, e.g. creating and publishing datasets with many datafiles and using file DOIs, +or doing bulk operations that involve DataCite with many datasets. + +### New Settings + +- dataverse.feature.only-update-datacite-when-needed + +The default is false - Dataverse will not check to see if DataCite's information is out of date before sending an update. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 14bf33c9482..c96adcec01b 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -537,6 +537,8 @@ dataverse.pid.*.datacite.username ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ dataverse.pid.*.datacite.password ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +dataverse.feature.only-update-datacite-when-needed +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ PID Providers of type ``datacite`` require four additional parameters that define how the provider connects to DataCite. DataCite has two APIs that are used in Dataverse: @@ -552,6 +554,11 @@ for `Fabrica `_ and their APIs. You need to provide the same credentials (``username``, ``password``) to Dataverse software to mint and manage DOIs for you. As noted above, you should use one of the more secure options for setting the password. +The `only-update-datacite-when-needed feature` flag is a global option that causes Dataverse to GET the latest metadata from DataCite +for a DOI and compare it with the current metadata in Dataverse and only sending a following POST request if needed. This potentially +substitutes a read for an unnecessary write at DataCite, but would result in extra reads when all metadata in Dataverse is new. +Setting the flag to "true" is recommended when using DataCite file DOIs. + CrossRef-specific Settings ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -3805,6 +3812,9 @@ please find all known feature flags below. Any of these flags can be activated u * - enable-pid-failure-log - Turns on creation of a monthly log file (logs/PIDFailures_.log) showing failed requests for dataset/file PIDs. Can be used directly or with scripts at https://github.com/gdcc/dataverse-recipes/python/pid_reports to alert admins. - ``Off`` + * - only-update-datacite-when-needed + - Only contact DataCite to update a DOI after checking to see if DataCite has outdated information (for efficiency, lighter load on DataCite, especially when using file DOIs). + - ``Off`` **Note:** Feature flags can be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_FEATURE_XXX`` (e.g. ``DATAVERSE_FEATURE_API_SESSION_AUTH=1``). These environment variables can be set in your shell before starting Payara. If you are using :doc:`Docker for development `, you can set them in the `docker compose `_ file. diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java index 0e4bb510f4b..76ea4941f96 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java @@ -232,7 +232,16 @@ public enum FeatureFlags { /** * Only update a DataCite DOI when needed (for efficiency, lighter load on DataCite). - */ + * This flag causes Dataverse to GET the latest metadata from DataCite for a DOI and + * comparing it with the current metadata in Dataverse and only sending a following POST + * request if needed. This potentially substitutes a read for an unnecessary write at DataCite, + * but would result in extra reads when all metadata in Dataverse is new. Setting the flag + * to "true" is recommended when using DataCite file DOIs. + * + * @apiNote Raise flag by setting + * "dataverse.feature.only-update-datacite-when-needed" + * @since Dataverse 6.9 + */ ONLY_UPDATE_DATACITE_WHEN_NEEDED("only-update-datacite-when-needed"), ; From d4dbd3a23cec5a82d791d2e5a33ed1e0c50257c6 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 19 Sep 2025 16:24:51 -0400 Subject: [PATCH 287/634] #11710 add search term to already linked --- .../iq/dataverse/DatasetLinkingDataverse.java | 13 ++++++ .../dataverse/DatasetLinkingServiceBean.java | 45 ++++++++++++++++--- .../impl/GetLinkingDataverseListCommand.java | 2 +- .../iq/dataverse/api/DataversesIT.java | 10 ++++- .../edu/harvard/iq/dataverse/api/UtilIT.java | 2 +- 5 files changed, 64 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetLinkingDataverse.java b/src/main/java/edu/harvard/iq/dataverse/DatasetLinkingDataverse.java index dec07a09643..28b061ffa2a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetLinkingDataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetLinkingDataverse.java @@ -9,6 +9,7 @@ import jakarta.persistence.Id; import jakarta.persistence.Index; import jakarta.persistence.JoinColumn; +import jakarta.persistence.NamedNativeQuery; import jakarta.persistence.NamedQueries; import jakarta.persistence.NamedQuery; import jakarta.persistence.OneToOne; @@ -35,6 +36,18 @@ @NamedQuery(name = "DatasetLinkingDataverse.findIdsByLinkingDataverseId", query = "SELECT o.dataset.id FROM DatasetLinkingDataverse AS o WHERE o.linkingDataverse.id = :linkingDataverseId") }) + + @NamedNativeQuery( + name = "DatasetLinkingDataverse.findByDatasetIdAndLinkingDataverseName", + query = """ + select o.linkingDataverse_id from DatasetLinkingDataverse as o + LEFT JOIN dataverse dv ON dv.id = o.linkingDataverse_id + WHERE o.dataset_id =? AND ((LOWER(dv.name) LIKE ? and ((SUBSTRING(LOWER(dv.name),0,(LENGTH(dv.name)-9)) LIKE ?) + or (SUBSTRING(LOWER(dv.name),0,(LENGTH(dv.name)-9)) LIKE ?))) + or (LOWER(dv.name) NOT LIKE ? and ((LOWER(dv.name) LIKE ?) + or (LOWER(dv.name) LIKE ?))))""" + ) + public class DatasetLinkingDataverse implements Serializable { private static final long serialVersionUID = 1L; @Id diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetLinkingServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetLinkingServiceBean.java index 514ecb1dc4d..eb54d239e11 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetLinkingServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetLinkingServiceBean.java @@ -5,6 +5,7 @@ */ package edu.harvard.iq.dataverse; +import jakarta.ejb.EJB; import java.util.ArrayList; import java.util.List; import java.util.logging.Logger; @@ -27,6 +28,9 @@ public class DatasetLinkingServiceBean implements java.io.Serializable { @PersistenceContext(unitName = "VDCNet-ejbPU") private EntityManager em; + @EJB + DataverseServiceBean dataverseService; + public List findLinkedDatasets(Long dataverseId) { @@ -40,13 +44,44 @@ public List findLinkedDatasets(Long dataverseId) { } public List findLinkingDataverses(Long datasetId) { + return findLinkingDataverses(datasetId, ""); + } + + public List findLinkingDataverses(Long datasetId, String searchTerm) { List retList = new ArrayList<>(); - TypedQuery typedQuery = em.createNamedQuery("DatasetLinkingDataverse.findByDatasetId", DatasetLinkingDataverse.class) - .setParameter("datasetId", datasetId); - for (DatasetLinkingDataverse datasetLinkingDataverse : typedQuery.getResultList()) { - retList.add(datasetLinkingDataverse.getLinkingDataverse()); + if (searchTerm == null || searchTerm.isEmpty()) { + TypedQuery typedQuery = em.createNamedQuery("DatasetLinkingDataverse.findByDatasetId", DatasetLinkingDataverse.class) + .setParameter("datasetId", datasetId); + for (DatasetLinkingDataverse datasetLinkingDataverse : typedQuery.getResultList()) { + retList.add(datasetLinkingDataverse.getLinkingDataverse()); + } + return retList; + + } else { + + String pattern = searchTerm.toLowerCase(); + + String pattern1 = pattern + "%"; + String pattern2 = "% " + pattern + "%"; + + // Adjust the queries for very short, 1 and 2-character patterns: + if (pattern.length() == 1) { + pattern1 = pattern; + pattern2 = pattern + " %"; + } + System.out.print("pattern1: " + pattern1); + System.out.print("pattern2: " + pattern2); + TypedQuery typedQuery + = em.createNamedQuery("DatasetLinkingDataverse.findByDatasetIdAndLinkingDataverseName", Long.class) + .setParameter(1, datasetId).setParameter(2, "%dataverse").setParameter(3, pattern1) + .setParameter(4, pattern2).setParameter(5, "%dataverse").setParameter(6, pattern1).setParameter(7, pattern2); + + for (Long id : typedQuery.getResultList()) { + retList.add(dataverseService.find(id)); + } + return retList; } - return retList; + } public void save(DatasetLinkingDataverse datasetLinkingDataverse) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java index 793175d26b7..c20e545982c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java @@ -86,7 +86,7 @@ public List execute(CommandContext ctxt) throws CommandException { return ctxt.dataverses().removeUnlinkableDataverses(dataversesForLinking, dvObject); } else { if (dvObject instanceof Dataset) { - return ctxt.dsLinking().findLinkingDataverses(dvObject.getId()); + return ctxt.dsLinking().findLinkingDataverses(dvObject.getId(), searchParam); } else { return ctxt.dvLinking().findLinkingDataverses(dvObject.getId()); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 86b082d79ae..fe9d9b74de0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -842,7 +842,15 @@ public void testGetLinkableDataverses(){ getLinkableDataverses.prettyPrint(); getLinkableDataverses.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data.size()", equalTo(1)); + .body("data.size()", equalTo(1)); + + //if you ask for already linked you should get one unless there's a bad search terms + getLinkableDataverses = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiToken, "QQQBatmanSymbol", true); + + getLinkableDataverses.prettyPrint(); + getLinkableDataverses.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", equalTo(0)); //set user api back to super user for cleanup UtilIT.setSuperuserStatus(username, Boolean.TRUE); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 47b8deaf0e4..83c14a84a76 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4749,7 +4749,7 @@ static Response getLinkableDataverses(String type, String dvObjectId, String api } if (alreadyLinked){ - if (optionalQueryParam.isEmpty()){ + if (optionalQueryParam.isEmpty() && (searchTerm == null || searchTerm.isEmpty())){ optionalQueryParam = "?alreadyLinking=true"; } else { optionalQueryParam = optionalQueryParam + "&alreadyLinking=true"; From b63555f0941b9993a722d06d6b1d44602eaf5ca9 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 22 Sep 2025 10:26:48 -0400 Subject: [PATCH 288/634] #11710 add perms to find already linked --- .../dataverse/DatasetLinkingServiceBean.java | 2 - .../dataverse/DataverseLinkingDataverse.java | 11 ++++++ .../DataverseLinkingServiceBean.java | 37 +++++++++++++++++-- .../impl/GetLinkingDataverseListCommand.java | 23 +++++++++++- .../iq/dataverse/api/DataversesIT.java | 4 +- 5 files changed, 67 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetLinkingServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetLinkingServiceBean.java index eb54d239e11..6b0f8af6590 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetLinkingServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetLinkingServiceBean.java @@ -69,8 +69,6 @@ public List findLinkingDataverses(Long datasetId, String searchTerm) pattern1 = pattern; pattern2 = pattern + " %"; } - System.out.print("pattern1: " + pattern1); - System.out.print("pattern2: " + pattern2); TypedQuery typedQuery = em.createNamedQuery("DatasetLinkingDataverse.findByDatasetIdAndLinkingDataverseName", Long.class) .setParameter(1, datasetId).setParameter(2, "%dataverse").setParameter(3, pattern1) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseLinkingDataverse.java b/src/main/java/edu/harvard/iq/dataverse/DataverseLinkingDataverse.java index 3030922ea5e..bf2326f0e06 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseLinkingDataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseLinkingDataverse.java @@ -13,6 +13,7 @@ import jakarta.persistence.Id; import jakarta.persistence.Index; import jakarta.persistence.JoinColumn; +import jakarta.persistence.NamedNativeQuery; import jakarta.persistence.NamedQueries; import jakarta.persistence.NamedQuery; import jakarta.persistence.OneToOne; @@ -39,6 +40,16 @@ @NamedQuery(name = "DataverseLinkingDataverse.findIdsByLinkingDataverseId", query = "SELECT o.dataverse.id FROM DataverseLinkingDataverse AS o WHERE o.linkingDataverse.id = :linkingDataverseId") }) + @NamedNativeQuery( + name = "DataverseLinkingDataverse.findByDataverseIdAndLinkingDataverseName", + query = """ + select o.linkingDataverse_id from DataverseLinkingDataverse as o + LEFT JOIN dataverse dv ON dv.id = o.linkingDataverse_id + WHERE o.dataverse_id =? AND ((LOWER(dv.name) LIKE ? and ((SUBSTRING(LOWER(dv.name),0,(LENGTH(dv.name)-9)) LIKE ?) + or (SUBSTRING(LOWER(dv.name),0,(LENGTH(dv.name)-9)) LIKE ?))) + or (LOWER(dv.name) NOT LIKE ? and ((LOWER(dv.name) LIKE ?) + or (LOWER(dv.name) LIKE ?))))""" + ) public class DataverseLinkingDataverse implements Serializable { private static final long serialVersionUID = 1L; @Id diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseLinkingServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseLinkingServiceBean.java index 834ff96e392..9f1bcde4c0e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseLinkingServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseLinkingServiceBean.java @@ -43,12 +43,41 @@ public List findLinkedDataverses(Long linkingDataverseId) { } public List findLinkingDataverses(Long dataverseId) { + + return findLinkingDataverses(dataverseId, ""); + } + + public List findLinkingDataverses(Long dataverseId, String searchTerm) { List retList = new ArrayList<>(); - TypedQuery typedQuery = em.createNamedQuery("DataverseLinkingDataverse.findByDataverseId", DataverseLinkingDataverse.class) - .setParameter("dataverseId", dataverseId); - for (DataverseLinkingDataverse dataverseLinkingDataverse : typedQuery.getResultList()) { - retList.add(dataverseLinkingDataverse.getLinkingDataverse()); + if (searchTerm == null || searchTerm.isEmpty()) { + TypedQuery typedQuery = em.createNamedQuery("DataverseLinkingDataverse.findByDataverseId", DataverseLinkingDataverse.class) + .setParameter("dataverseId", dataverseId); + for (DataverseLinkingDataverse dataverseLinkingDataverse : typedQuery.getResultList()) { + retList.add(dataverseLinkingDataverse.getLinkingDataverse()); + } + + } else { + + String pattern = searchTerm.toLowerCase(); + + String pattern1 = pattern + "%"; + String pattern2 = "% " + pattern + "%"; + + // Adjust the queries for very short, 1 and 2-character patterns: + if (pattern.length() == 1) { + pattern1 = pattern; + pattern2 = pattern + " %"; + } + TypedQuery typedQuery + = em.createNamedQuery("DataverseLinkingDataverse.findByDataverseIdAndLinkingDataverseName", Long.class) + .setParameter(1, dataverseId).setParameter(2, "%dataverse").setParameter(3, pattern1) + .setParameter(4, pattern2).setParameter(5, "%dataverse").setParameter(6, pattern1).setParameter(7, pattern2); + + for (Long id : typedQuery.getResultList()) { + retList.add(dataverseService.find(id)); + } } + return retList; } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java index c20e545982c..077c420d669 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java @@ -14,6 +14,8 @@ import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.util.BundleUtil; +import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; /** @@ -85,11 +87,28 @@ public List execute(CommandContext ctxt) throws CommandException { } return ctxt.dataverses().removeUnlinkableDataverses(dataversesForLinking, dvObject); } else { + List dataversesAlreadyLinked; + List dataversesAlreadyLinkedCanUnlink = new ArrayList<>(); + //new ArrayList<>(); if (dvObject instanceof Dataset) { - return ctxt.dsLinking().findLinkingDataverses(dvObject.getId(), searchParam); + dataversesAlreadyLinked = ctxt.dsLinking().findLinkingDataverses(dvObject.getId(), searchParam); + for (Dataverse dv : dataversesAlreadyLinked) { + if (ctxt.permissions().hasPermissionsFor(getRequest(), dv, EnumSet.of(Permission.LinkDataset))) { + dataversesAlreadyLinkedCanUnlink.add(dv); + } + return dataversesAlreadyLinkedCanUnlink; + } + //this.permissionService.requestOn(req, res).has(Permission.LinkDataverse) } else { - return ctxt.dvLinking().findLinkingDataverses(dvObject.getId()); + dataversesAlreadyLinked = ctxt.dvLinking().findLinkingDataverses(dvObject.getId(), searchParam); + for (Dataverse dv : dataversesAlreadyLinked) { + if (ctxt.permissions().hasPermissionsFor(getRequest(), dv, EnumSet.of(Permission.LinkDataverse))) { + dataversesAlreadyLinkedCanUnlink.add(dv); + } + } + return dataversesAlreadyLinkedCanUnlink; } + return null; } } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index fe9d9b74de0..9423c475499 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -850,8 +850,8 @@ public void testGetLinkableDataverses(){ getLinkableDataverses.prettyPrint(); getLinkableDataverses.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data.size()", equalTo(0)); - + .body("data.size()", equalTo(0)); + //set user api back to super user for cleanup UtilIT.setSuperuserStatus(username, Boolean.TRUE); From 44eec3b74cfca862f56a0fb50c121cb3dc0d5665 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 22 Sep 2025 10:58:38 -0400 Subject: [PATCH 289/634] #11710 add test for without perms --- .../command/impl/GetLinkingDataverseListCommand.java | 3 +-- .../java/edu/harvard/iq/dataverse/api/DataversesIT.java | 8 ++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java index 077c420d669..466871b88e4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLinkingDataverseListCommand.java @@ -77,7 +77,7 @@ public List execute(CommandContext ctxt) throws CommandException { } else { permToCheck = Permission.LinkDataverse; } - //dependin on the already linked boolean the command will return a list of Dataverses available for linking + //depending on the already linked boolean the command will return a list of Dataverses available for linking // or a list of dataverses to which the object has already been linked - for the unlink function if (!alreadyLinked) { dataversesForLinking = ctxt.permissions().findPermittedCollections(getRequest(), authUser, permToCheck, searchParam); @@ -98,7 +98,6 @@ public List execute(CommandContext ctxt) throws CommandException { } return dataversesAlreadyLinkedCanUnlink; } - //this.permissionService.requestOn(req, res).has(Permission.LinkDataverse) } else { dataversesAlreadyLinked = ctxt.dvLinking().findLinkingDataverses(dvObject.getId(), searchParam); for (Dataverse dv : dataversesAlreadyLinked) { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 9423c475499..719c820f1de 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -844,6 +844,14 @@ public void testGetLinkableDataverses(){ .statusCode(OK.getStatusCode()) .body("data.size()", equalTo(1)); + //if you ask for already linked you should get one unless you don't have perms + getLinkableDataverses = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiTokenThree, dataverseAliasForLinking, true); + + getLinkableDataverses.prettyPrint(); + getLinkableDataverses.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", equalTo(0)); + //if you ask for already linked you should get one unless there's a bad search terms getLinkableDataverses = UtilIT.getLinkableDataverses("dataset", datasetPersistentId, apiToken, "QQQBatmanSymbol", true); From 3f63e273829ae9b20d210c14403c2b1fbecab5a9 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 22 Sep 2025 13:31:33 -0400 Subject: [PATCH 290/634] more changes --- .../source/developers/big-data-support.rst | 260 ++++++++++++++---- .../source/installation/config.rst | 28 +- 2 files changed, 217 insertions(+), 71 deletions(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index cf156bd12c8..1c388a7c565 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -8,11 +8,10 @@ work differently than Dataverse's default configuration, potentially requiring u In general, there are three dimensions in which Dataverse can scale: -1. **Number of Datasets** +1. **Storage size of individual files and aggregate storage size** 2. **Number of Files per Dataset** -3. **Storage size of individual files and aggregate storage size** +3. **Number of Datasets** -This guide will primarily focus on the latter two dimensions as the main way to scale the number of datasets is to scale the database, server, and storage resources. .. contents:: |toctitle| :local: @@ -20,7 +19,8 @@ This guide will primarily focus on the latter two dimensions as the main way to Choosing the Right Storage -------------------------- -Dataverse provides several options that affect how files are transferred and stored which significantly impact the performance and scalability of your Dataverse installation. +The main issues for handling larger files and larger aggregate data size relate to the performance of the storage used and how involved the Dataverse server is in the data transfer. +With appropriate configuration, Dataverse can support file sizes and aggregate dataset sizes into the terabyte scale and beyond. File Storage ~~~~~~~~~~~~ @@ -28,29 +28,33 @@ File Storage The default storage option in Dataverse is based on a local file system. When files are transferred to Dataverse, they are first stored in a temporary location on the Dataverse server. Any zip files uploaded are unzipped to create multiple individual file entries. Once an upload is completed, Dataverse copies the files to permanent storage. Dataverse also takes advantage of the file being local to inspect it's bytes to determine it -s MIME type, and, for tabular data, to 'ingest' it - extracting metadata about the variables used in the file and creating a tab-separated value (TSV) version of the file. +s MIME type, and, for tabular data, to 'ingest' it - extracting metadata about the variables used in the file and creating a tab-separated value (TSV) version of the file. -Benefits: This option requires no external services and can potentially handle files into the gigabyte (GB) size range. FOr smaller institutions, +Benefits: + +- This option requires no external services and can potentially handle files into the gigabyte (GB) size range. For smaller institutions, and in disciplines where datasets do not have more than a few hundred files and files are not too large, this can be the simplest option. -Further, unzipping of zip archives can be simpler for users than having to upload many individual files and was, at one time, the only way to +- Unzipping of zip archives can be simpler for users than having to upload many individual files and was, at one time, the only way to preserve file path names when uploading. Challenges: In general file storage is not a good option for larger data sizes - both in terms of file size and number of files. Contributing factors include: -- Local file storage must be provisioned in advance based on anticipated demand. It can involve up-front costs (for a local disk), or, when procured from a cloud provider, is likely to be more expensive than object storage (see below) - Because temporary storage is used, transfers will temporarily use several times as much space as the final transfer. Unzipping also increases the final storage size of a dataset. - Because all uploads use the same temporary storage, temporary storage must be large enough to handle multiple users uploading data. - Each file is uploaded as a single HTTP request, which can cause long transfer times which, in turn, can trigger timeout errors in Dataverse or any proxy or load balancer in front of Dataverse. -- Uploading many files at once can trigger any rate limiter (i.e. used to throttle use by AI) resulting in failures +- Uploading many files at once can trigger any rate limiter in front of the Dataverse server (i.e. used to throttle use by AI) resulting in failures - Because transfers (both uploads and downloads) are handled by the Dataverse server, they add to server processing load which can affect overall performance. +- Cost: local file storage must be provisioned in advance based on anticipated demand. It can involve up-front costs (for a local disk), or, when procured from a cloud provider, is likely to be more expensive than object storage (see below) + Object Storage via S3 ~~~~~~~~~~~~~~~~~~~~~ A more scalable option for storage is to use an object store with transfers managed using the Simple Storage Service (S3) protocol. S3-compatible storage can easily be bought (rented) from major cloud providers, but may also be available from institutional clouds. It is also possible to run open-source software to provide S3 storage -over a local file system (making it possible to enjoy the advantages discussed below while still leveraging local file storage). While Dataverse can be -configured to handle uploads and downloads as with file storage (with zip files being unzipped, but having many of the same challenges in terms of temporary storage and server load as discussed above) +over a local file system (making it possible to enjoy the advantages discussed below while still leveraging local file storage). + +While Dataverse can be configured to handle uploads and downloads as with file storage (with zip files being unzipped, but having many of the same challenges in terms of temporary storage and server load as discussed above) it can also be configured to use 'direct' upload and download. In this configuration, the actual transfer of file bytes is from/to the user's local machine to/from the S3 store. In this configuration, Dataverse does not attempt to unzip zip files and they are stored as a single file in the dataset. @@ -80,33 +84,40 @@ upload them, with their relative paths intact, to dataverse. (DVWebloader can on standard upload interface in Dataverse). - Using direct upload stops Dataverse from inspecting the file bytes to determine the MIME type (with one exception - Stata files). Dataverse will still look at the file name and extension to determine the MIME type. - To perform the 'ingest' processing, Dataverse currently has to copy the file to local storage, somewhat negating the benefit of sending data directly to S3. To manage larger files, one can set a per-store -Ingest size limit (which can be 0 bytes) to stop ingest or limit it to smaller files. +ingest size limit (which can be 0 bytes) to stop ingest or limit it to smaller files. - Dataverse's mechanism for downloading a whole dataset or multiple selected files involves zipping those files together. Even When using S3 with direct upload/download, the file bytes are transferred to the Dataverse server as part of the zipping process. There are ways to reduce the performance impact of this: - There is a 'ZipDownloader' app that can be run separate from Dataverse to handle the zipping process. - Dataverse has a :ZipDownloadLimit that can be used to limit the amount of data that can be zipped. If a dataset is larger than this limit, Dataverse will only add some of the files to the zip and list others in the included manifest file. - There are tools such as the Dataverse Dataset Downloader (https://github.com/gdcc/dataverse-recipes/tree/main/shell/download#dataverse-dataset-downloader) that can be used to download all off the files individually. This avoids sending any of the files to/through the Dataverse server when S3 direct download is enabled. - Dataverse leverages S3 features that are not implemented by all servers and has several configuration options geared towards handling variations between servers. Site admins should be sure to test with their preferred S3 implementation. +- The part-size used when directly transferring files to S3 is configurable (at AWS, from 5 MiB to 5GiB). The default in Dataverse is 1 GiB (1024^3). If the primary use case is with smaller files than that, decreasing the part size may improve upload speeds. Remote Storage ~~~~~~~~~~~~~~ Note: Remote Storage is still experimental: feedback is welcome! See :ref:`support`. -For very large, and/or very sensitive data, it may not make sense to transfer or copy files to Dataverse at all. The ``remote`` store type in the Dataverse software support these use cases. +For very large, and/or very sensitive data, it may not make sense to transfer or copy files to Dataverse at all. +The ``remote`` store type in the Dataverse software supports these use cases. It allows Dataverse to store a URL reference for the file rather than transferring the file bytes to a store managed directly by Dataverse. -In the most basic configuration a site administrator configures the base URL for the store, e.g. "https://thirdpartystorage.edu/long-term-storage/" and users can then create files referencing -any URL starting with that base, e.g. "https://thirdpartystorage.edu/long-term-storage/my_file.txt". If the remote site is a public web server, the remote store in Dataverse should be configured to be 'public' which will -disable the ability to restrict or embargo files (as they are public on the remote site and Dataverse cannot block access.) Conversely, Dataverse can be configured to sign requests to the remote server and, -and, which the remote server, if it is capable of validating them, can use to reject requests not approved by Dataverse. In this configuration, users can restrict and embargo files and Dataverse and the remote server will cooperate to -manage access control. Another alternative, with a more advanced remote store, would be, instead of using URLs that directly enable download of the file, to use URLs that point to -a landing page at the remote server that may require the user to login, or go through some other authentication/validation process before being able to access the file. - -Dataverse considers remote storage to be read-only, or, in cases where the remote service does not provide a way for Dataverse to download the file bytes ((due to access control or because the URL refers to a landing page), inacessible. -Depending on whether Dataverse can access the bytes of the file, functionality such as ingest and integrity checking may or may not be possible. If the file bytes are not accessible -the remote store in Dataverse should be configured to disable operations that attempt to access the file (see files-not-accessible-by-dataverse). -If the file bytes are accessible, Dataverse can still support features such as ingest and thumbnail creation, as well as local storage of other files and auxilliary files. These are handled by configuring a 'base' store with the remote store -that is used for these purposes. (This means that while the specified files remain on the remote store, other files in the dataset, and potentially the ingested TSV format of a remote file would be managed by Dataverse in some other store. If ingest is not desired, the ingest size limit for the store can be set to 0 bytes). +In the most basic configuration a site administrator configures the base URL for the store, e.g. "https://thirdpartystorage.edu/long-term-storage/" +and users can then create files referencing any URL starting with that base, e.g. "https://thirdpartystorage.edu/long-term-storage/my_project_dir/my_file.txt". +If the remote site is a public web server, the remote store in Dataverse should be configured to be 'public' which will disable the ability to restrict +or embargo files (as they are public on the remote site and Dataverse cannot block access.) Conversely, Dataverse can be configured to sign requests to the +remote server which the remote server can then, if it is capable of validating them, use to reject requests not approved by Dataverse. In this configuration, +users can restrict and embargo files and Dataverse and the remote server will cooperate to manage access control. Another alternative, with a more advanced +remote store, would be, instead of using URLs that directly enable download of the file, to use URLs that point to a landing page at the remote server that +may require the user to login, or go through some other authentication/validation process before being able to access the file. + +Dataverse considers remote storage to be read-only, or, in cases where the remote service does not provide a way for Dataverse to download the file bytes +(due to access control or because the URL refers to a landing page), inacessible. Depending on whether Dataverse can access the bytes of the file, +functionality such as ingest and integrity checking may or may not be possible. If the file bytes are not accessible the remote store in Dataverse should be +configured to disable operations that attempt to access the file (see files-not-accessible-by-dataverse). If the file bytes are accessible, Dataverse can still +support features such as ingest and thumbnail creation, as well as local storage of other files and auxilliary files. These are handled by configuring a 'base' store +with the remote store that is used for these purposes. (This means that while the specified files remain on the remote store, other files in the dataset, and +potentially the ingested TSV format of a remote file would be managed by Dataverse in some other store. If ingest is not desired, the ingest size limit for the +store can be set to 0 bytes). Benefits: @@ -116,23 +127,24 @@ Benefits: Challenges: - Currently, remote files can only be added via the API. (This may be addressed in future versions). +- Since remote files can only be added as the dataset is created in the UI. However, the UI will still allow upload of files to the base store (at dataset creation and when editing) - As Dataverse is relying on the remote service to main the integrity and availability of the files, it is likely that the Dataverse site admin will want to have a formal agreement with the remote service operator about their policies. - Site admins need to consider carefully how to configure file size limits, ingest size limits, etc. on the remote store and it's base store, and whether the remote store is public-only, and whether it files there can be read by Dataverse to assure the requirements of a specific use case(s) are addressed. - The current remote store implementation will not prevent you from providing a relative URL that results in a 404 when resolved. (I.e. if you make a typo). You should check to make sure the file exists at the location you specify - by trying to download in Dataverse, by checking to see that Dataverse was able to get the file size (which it does by doing a HEAD call to that location), or just manually trying the URL in your browser. - For large files, direct-download should always be used with a remote store. (Otherwise the Dataverse will be involved in the download.) - +- When multiple files are selected for download, Dataverse will try to include remote files in the zip file being created (up to the max zip size limit) which is inefficent, and will not be able to include remote files that are inaccessible (possibly confusing). Globus Transfer ~~~~~~~~~~~~~~~ Note: Globus Transfer is still experimental: feedback is welcome! See :ref:`support`. -`Globus `_ provides file transfer service that is widely used for the largest datasets (in terms of both file size and number of files). It provides: +`Globus `_ provides file transfer service that is widely used for the world's largest datasets (in terms of both file size and number of files). It provides: - robust file transfer capable of handling delays (e.g. due to the time it takes to mount tapes) and restarting after network or endpoint failures -- parallel file transfers, potentially between clusters of computers on both ends +- rapid parallel file transfers, potentially between clusters of computers on both ends - third-party transfer, which enables a user working with their desktop browser to initiate transfer of files from one remote endpoint (i.e. on a local high-performance computing cluster) to/from another (e.g. one associated with a Dataverse store) Dataverse can be configured to support Globus transfers in multiple ways: @@ -141,19 +153,20 @@ Dataverse can be configured to support Globus transfers in multiple ways: - A Dataverse-managed Globus S3 Endpoint: Dataverse controls user access to the endpoint, access is available via S3 and via Globus - A Globus Endpoint treated as Remote Storage: Dataverse references files on a Globus endpoint managed by a third party -Each of these options has its own advantages and disadvantages: Benefits: - Globus scales to higher data volumes than any other option. Users working with large data are often familiar with Globus and are interested in transferring data to/from computational clusters rather than their local machine - Globus transfers can be initiated by choosing the Globus option in the dataset upload panel. Analogously, "Globus Transfer" is one of the download options in the "Access Dataset" menu. -- For the non-S3 options, Dataverse support having a base store (e.g. a local file system or an S3-based store), which can be used internally by Dataverse (e.g. for thumbnails, etc.) and can allow users to upload smaller files (e.g. Readmes, documentation) that might not be suited to a given Globus endpoint (e.g. a tape store) +- For the non-S3 options, Dataverse supports having a base store (e.g. a local file system or an S3-based store), which can be used internally by Dataverse (e.g. for thumbnails, etc.) and can allow users to upload smaller files (e.g. Readmes, documentation) that might not be suited to a given Globus endpoint (e.g. a tape store) Challenges: + - Globus is complex to manage and Dataverse installations will need to develop Globus expertise or partner with another organization (i.e. a institutional high-performance computing center) to manage Globus endpoints. -- For users not familiar with Globus, managing transfers can be confusing. Globus does provide a free 'Globus Personal Connect' service which can be run on any machine to allow transfers to/from it. +- For users not familiar with Globus, managing transfers can be confusing. For the non-S3 options, users cannot just download files - they must have access to a destination Globus endpoint. Globus does provide a free 'Globus Personal Connect' service which installed on any machine to allow transfers to/from it. - Globus transfers are not enabled at dataset-creation time. Once the draft version is created, users can initiate Globus transfers to upload files from remote endpoints. -- For Dataverse managed endpoints, a a community-developed `dataverse-globus `_ app must be installed and configured in the Dataverse instance. +- For Dataverse-managed endpoints, a community-developed `dataverse-globus `_ app must be installed and configured in the Dataverse instance. This app manages granting and revoking access for users to upload/download files from Dataverse and handles the translation between Dataverse's internal file naming/organization to that see by the user. +- Users familiar with Globus sometimes expect to be able to find the Dataverse endpoint in Globus' online service and download files from there. Due to the fact that Dataverse is managing permissions and handling file naming, this doesn't work. - Due to differences between Dataverse's and Globus's access control models, Dataverse cannot enforce per-file-access restrictions - restriction can only be done today at the level of providing access to all files in a dataset. Globus stores can be defined as public to disable Dataverse's ability to restrict and embargo files in that store. If the store is configured to support restriction and embargo, Dataverse and it's Dataverse-Globus app will limit users to downloading only the files they have been granted access to, but a technically knowledgeable user could access other files in the same dataset if they are give access to one. @@ -161,50 +174,181 @@ Dataverse and it's Dataverse-Globus app will limit users to downloading only the - Dataverse-managed endpoints must be Globus 'guest collections' hosted on either a file-system-based endpoint or an S3-based endpoint (the latter requires use of the Globus S3 connector which requires a paid Globus subscription at the host institution). In either case, Dataverse is configured with the Globus credentials of a user account that can manage the endpoint. Users will need their own Globus account, which can be obtained via their institution or directly from Globus (at no cost). -- With the file-system endpoint, Dataverse does not currently have access to the file contents. Thus, functionality related to ingest, previews, fixity hash validation, etc. are not available. (Using the S3-based endpoint, Dataverse has access via S3 and all functionality normally associated with direct uploads to S3 is available.) +- With the file-system endpoint, Dataverse does not currently have access to the file contents. Thus, functionality related to ingest, previews, fixity hash validation, etc. are not available. +(Using the S3-based endpoint, Dataverse has access via S3 and all functionality normally associated with direct uploads to S3 is available. In this case admins should be sure to set the maximum size for ingest and avoid requiring hash validation at publication, etc.) - For the reference use case, Dataverse must be configured with a list of allowed endpoint/base paths from which files may be referenced. In this case, since Dataverse is not accessing the remote endpoint itself, it does not need Globus credentials. -Users will also need a Globus account in this case, and the remote endpoint must be configured to allow them access (i.e. be publicly readable, or potentially involving some out-of-band mechanism to request access (that could be described in the dataset's Terms of Use and Access). +Users will also need a Globus account in this case, and the remote endpoint must be configured to allow them access (i.e. be publicly readable, or potentially involving some out-of-band mechanism to request access (that could be described, for example, in the dataset's Terms of Use and Access). - While Globus itself can handle many (millions of) files of any size, Dataverse cannot handle more than thousands of files per dataset (at best) and some Globus endpoints may have limits on file sizes - both maximums and minimums (e.g. for tape storage where small files are inefficient). -Users will need to be made aware of these limitations and the possibilities for managing them (e.g. by aggregating multiple files in a single larger file). +Users will need to be made aware of these limitations and the possibilities for managing them (e.g. by aggregating multiple files in a single larger file, or storing smaller files in the base-store via the normal Dataverse upload UI). +- There is currently (a bug)[https://github.com/gdcc/dataverse-globus/issues/2] that won't allow users to transfer files from/to endpoints where they do not have permission to list the overall file tree (i.e. an institution manages /institution_name but the user only has access to /institution_name/my_dir.) +Until that is fixed, a work-around is to first transfer data to an endpoint without this restriction. +- An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. While it is now the recommended option, it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag (see :ref:`feature-flags`) and the JVM option :ref:`dataverse.files.globus-monitoring-server`. More details of the setup required to enable Globus is described in the `Community Dataverse-Globus Setup and Configuration document `_ and the references therein. -As described in that document, - An overview of the control and data transfer interactions between components was presented at the 2022 Dataverse Community Meeting and can be viewed in the `Integrations and Tools Session Video `_ around the 1 hr 28 min mark. See also :ref:`Globus settings <:GlobusSettings>`. -An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. Due to its experimental nature it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag (see :ref:`feature-flags`) and the JVM option :ref:`dataverse.files.globus-monitoring-server`. - - +Managing more files per dataset and more datasets +------------------------------------------------- +Dataverse can be configured to handle datasets with hundreds or thousands of files and hundreds of thousands of datasets. However, reaching these levels can require significant effort to appropriately configure the server. +Technically, there are two factors that can limit scaling the number of files per dataset: how much the Dataverse server is involved in data transfer, and constraints based on Dataverse's code and database configuration. +The former is dramatically affected by the choice for file storage and options such as the S3 direct upload/download settings and ingest size limits. The are fewer ways to affect the latter beyond increasing the amount of memory and CPU resources available +or rewriting the relevant parts of Dataverse. (There are continuing efforts to improve Dataverse's performance and scaling, so it is also advisable to use the latest version if you are pushing the boundaries on scaling. Progress is being made.) +Scaling to larger numbers of datasets (and to some extent scaling files per dataset) also depends on Dataverse's Solr search engine. There have been very significant improvements in indexing and search performance in recent releases, including some that are not turned on by default. +Avoiding Many Files +~~~~~~~~~~~~~~~~~~~ +Before describing things that can be done to improve scaling, it is important to note that there are configuration options and best practices to suggest to users to help avoid larger datasets and users hitting performance issues by going beyond the file counts you know work in your instance. + + /* zip upload number of files limit */ + ZipUploadFilesLimit, + /* the number of files the GUI user is allowed to upload in one batch, + via drag-and-drop, or through the file select dialog */ + MultipleUploadFilesLimit, + UseStorageQuotas, + /** + * Placeholder storage quota (defines the same quota setting for every user; used to test the concept of a quota. + */ + StorageQuotaSizeInBytes, + +Ways to Scale +~~~~~~~~~~~~~ + + + /** + * For published (public) objects, don't use a join when searching Solr. + * Experimental! Requires a reindex with the following feature flag enabled, + * in order to add the boolean publicObject_b:true field to all the public + * Solr documents. + * + * @apiNote Raise flag by setting + * "dataverse.feature.avoid-expensive-solr-join" + * @since Dataverse 6.3 + */ +AVOID_EXPENSIVE_SOLR_JOIN("avoid-expensive-solr-join"), + /** + * With this flag enabled, the boolean field publicObject_b:true will be + * added to all the indexed Solr documents for publicly-available collections, + * datasets and files. This flag makes it possible to rely on it in searches, + * instead of the very expensive join (the feature flag above). + * + * @apiNote Raise flag by setting + * "dataverse.feature.add-publicobject-solr-field" + * @since Dataverse 6.3 + */ + ADD_PUBLICOBJECT_SOLR_FIELD("add-publicobject-solr-field"), + /** + * Dataverse normally deletes all solr documents related to a dataset's files + * when the dataset is reindexed. With this flag enabled, additional logic is + * added to the reindex process to delete only the solr documents that are no + * longer needed. (Required docs will be updated rather than deleted and + * replaced.) Enabling this feature flag should make the reindex process + * faster without impacting the search results. + * + * @apiNote Raise flag by setting + * "dataverse.feature.reduce-solr-deletes" + * @since Dataverse 6.3 + */ + REDUCE_SOLR_DELETES("reduce-solr-deletes"), + /** + * This flag disables the feature that automatically selects one of the + * DataFile thumbnails in the dataset/version as the dedicated thumbnail + * for the dataset. + * + * @apiNote Raise flag by setting + * "dataverse.feature.disable-dataset-thumbnail-autoselect" + * @since Dataverse 6.4 + */ + DISABLE_DATASET_THUMBNAIL_AUTOSELECT("disable-dataset-thumbnail-autoselect"), + /** + * Only update a DataCite DOI when needed (for efficiency, lighter load on DataCite). + */ + ONLY_UPDATE_DATACITE_WHEN_NEEDED("only-update-datacite-when-needed"), + + +JvmSettings: + MIN_FILES_TO_USE_PROXY(SCOPE_SOLR, "min-files-to-use-proxy"), + + // INDEX CONCURENCY + SCOPE_SOLR_CONCURENCY(SCOPE_SOLR, "concurrency"), + MAX_ASYNC_INDEXES(SCOPE_SOLR_CONCURENCY, "max-async-indexes"), + + //EXPORTS + SCOPE_EXPORTS(PREFIX, "exports"), + SCOPE_EXPORTS_SCHEMA_DOT_ORG(SCOPE_EXPORTS, "schema-dot-org"), + EXPORTS_SCHEMA_DOT_ORG_MAX_FILES_FOR_DOWNLOAD_ENTRIES(SCOPE_EXPORTS_SCHEMA_DOT_ORG, "max-files-for-download-entries"), + +Settings: + + SolrFullTextIndexing, //true or false (default) + SolrMaxFileSizeForFullTextIndexing, //long - size in bytes (default unset/no limit) + /** Default Key for limiting the number of bytes uploaded via the Data Deposit API, UI (web site and . */ + MaxFileUploadSizeInBytes, + ZipDownloadLimit, +/* size limit for Tabular data file ingests */ + /* (can be set separately for specific ingestable formats; in which + case the actual stored option will be TabularIngestSizeLimit:{FORMAT_NAME} + where {FORMAT_NAME} is the format identification tag returned by the + getFormatName() method in the format-specific plugin; "sav" for the + SPSS/sav format, "RData" for R, etc. + for example: :TabularIngestSizeLimit:RData */ + TabularIngestSizeLimit, + /* Validate physical files in the dataset when publishing, if the dataset size less than the threshold limit */ + DatasetChecksumValidationSizeLimit, + /* Validate physical files in the dataset when publishing, if the datafile size less than the threshold limit */ + DataFileChecksumValidationSizeLimit, + FilePIDsEnabled, + /** + * If defined, this is the URL of the zipping service outside + * the main Application Service where zip downloads should be directed + * instead of /api/access/datafiles/ + */ + CustomZipDownloadServiceUrl, + + /** + * The URL for the DvWebLoader tool (see github.com/gdcc/dvwebloader for details) + */ + WebloaderUrl, + + + /** + * A comma-separated list of CategoryName in the desired order for files to be + * sorted in the file table display. If not set, files will be sorted + * alphabetically by default. If set, files will be sorted by these categories + * and alphabetically within each category. + */ + CategoryOrder, + /** + * True(default)/false option deciding whether ordering by folder should be applied to the + * dataset listing of datafiles. + */ + OrderByFolder, + /** + * Allows an instance admin to disable Solr search facets on the collection + * and dataset pages instantly + */ + DisableSolrFacets, + DisableSolrFacetsForGuestUsers, + DisableSolrFacetsWithoutJsession, + DisableUncheckedTypesFacet, + + /** + * When ingesting tabular data files, store the generated tab-delimited + * files *with* the variable names line up top. + */ + StoreIngestedTabularFilesWithVarHeaders, + + There is no well-defined cut-off - the speed of viewing and editing a large dataset will decrease as files are added, and, at some point, Dataverse may need more memory than is available, resulting in drastically slower response / -Handling Large Individual Files -------------------------------- - -When individual files exceed typical web upload/download capabilities (generally 1-2GB), you'll need specialized approaches. - -S3 Direct Upload and Download -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -**Best for:** Files up to several TB in size with minimal configuration changes - -A lightweight option for supporting file sizes beyond a few gigabytes - a size that can cause performance issues when uploaded through a Dataverse installation itself - is to configure an S3 store to provide direct upload and download via 'pre-signed URLs'. When these options are configured, file uploads and downloads are made directly to and from a configured S3 store using secure (https) connections that enforce a Dataverse installation's access controls. (The upload and download URLs are signed with a unique key that only allows access for a short time period and a Dataverse installation will only generate such a URL if the user has permission to upload/download the specific file in question.) - -This option can handle files >300GB and could be appropriate for files up to a TB or larger. Other options can scale farther, but this option has the advantages that it is simple to configure and does not require any user training - uploads and downloads are done via the same interface as normal uploads to a Dataverse installation. - -To configure these options, an administrator must set two JVM options for the Dataverse installation using the same process as for other configuration options: -``./asadmin create-jvm-options "-Ddataverse.files..download-redirect=true"`` -``./asadmin create-jvm-options "-Ddataverse.files..upload-redirect=true"`` -With multiple stores configured, it is possible to configure one S3 store with direct upload and/or download to support large files (in general or for specific Dataverse collections) while configuring only direct download, or no direct access for another store. The direct upload option now switches between uploading the file in one piece (up to 1 GB by default) and sending it as multiple parts. The default can be changed by setting: diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index a6aa44df974..369a58896aa 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1476,20 +1476,22 @@ Once you have configured a trusted remote store, you can point your users to the .. table:: :align: left - =========================================== ================== ========================================================================== =================== - JVM Option Value Description Default value - =========================================== ================== ========================================================================== =================== - dataverse.files..type ``remote`` **Required** to mark this storage as remote. (none) - dataverse.files..label **Required** label to be shown in the UI for this storage. (none) - dataverse.files..base-url **Required** All files must have URLs of the form /* . (none) - dataverse.files..base-store **Required** The id of a base store (of type file, s3, or swift). (the default store) - dataverse.files..download-redirect ``true``/``false`` Enable direct download (should usually be true). ``false`` - dataverse.files..secret-key A key used to sign download requests sent to the remote store. Optional. (none) - dataverse.files..url-expiration-minutes If direct downloads and using signing: time until links expire. Optional. 60 - dataverse.files..remote-store-name A short name used in the UI to indicate where a file is located. Optional. (none) - dataverse.files..remote-store-url A url to an info page about the remote store used in the UI. Optional. (none) + ======================================================= ================== ========================================================================== =================== + JVM Option Value Description Default value + ======================================================= ================== ========================================================================== =================== + dataverse.files..type ``remote`` **Required** to mark this storage as remote. (none) + dataverse.files..label **Required** label to be shown in the UI for this storage. (none) + dataverse.files..base-url **Required** All files must have URLs of the form /* . (none) + dataverse.files..base-store **Required** The id of a base store (of type file, s3, or swift). (the default store) + dataverse.files..upload-out-of-band ``true`` **Required to be true** Dataverse does not manage file placement ``false`` + dataverse.files..download-redirect ``true``/``false`` Enable direct download (should usually be true). ``false`` + dataverse.files..secret-key A key used to sign download requests sent to the remote store. Optional. (none) + dataverse.files..url-expiration-minutes If direct downloads and using signing: time until links expire. Optional. 60 + dataverse.files..remote-store-name A short name used in the UI to indicate where a file is located. Optional. (none) + dataverse.files..remote-store-url A url to an info page about the remote store used in the UI. Optional. (none) + dataverse.files..files-not-accessible-by-dataverse ``true``/``false`` True if the file is at the url provided, false if that is a landing page ``false`` - =========================================== ================== ========================================================================== =================== + ======================================================= ================== ========================================================================== =================== .. _globus-storage: From 5cbf6285442f0b83da159368e59d80ef9f97a7ad Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 22 Sep 2025 16:59:03 -0400 Subject: [PATCH 291/634] Update native-api.rst --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 1fba99039b6..f8f84430ac6 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -774,7 +774,7 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/dataverses/:persistentId/dataset/linkingDataverses?searchTerm=searchOn&persistentId=doi:10.5072/FK2/J8SJZB" - +You may also add an optional "alreadyLinked=true" parameter to return collections which are already linked to the given Dataset or Dataverse Collection. List Featured Collections for a Dataverse Collection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From b2d8e346f7f7cd8448c8e2eecffc95ad333ee04f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 23 Sep 2025 11:54:46 -0400 Subject: [PATCH 292/634] bump keycloak version --- conf/keycloak/builtin-users-spi/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/keycloak/builtin-users-spi/pom.xml b/conf/keycloak/builtin-users-spi/pom.xml index 8a4c6c5a5ad..2a730621f85 100644 --- a/conf/keycloak/builtin-users-spi/pom.xml +++ b/conf/keycloak/builtin-users-spi/pom.xml @@ -100,7 +100,7 @@ - 26.3.3 + 26.3.4 17 3.2.0 0.4 From ea2deb96d8fca5f3b0f7347a2b8b20c593d59042 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 25 Sep 2025 12:51:29 -0400 Subject: [PATCH 293/634] update base image version for post-6.8 release #11680 --- modules/dataverse-parent/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 013034f14d1..80d4ceaabe6 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -454,8 +454,8 @@ Once the release has been made (tag created), change this back to "${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}" (These properties are provided by the build-helper plugin below.) --> - - ${revision} + ${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion} + ${base.image.version} From 26818a76ae19ed7493902dbf3adc2625f01dce76 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 25 Sep 2025 13:29:41 -0400 Subject: [PATCH 294/634] #11772 prelim check in --- .../tests/data/update-dataset-terms.json | 20 +++++++ .../harvard/iq/dataverse/api/Datasets.java | 33 +++++++++++ .../impl/UpdateDatasetTermsOfUseCommand.java | 59 +++++++++++++++++++ .../impl/UpdateDatasetVersionCommand.java | 3 +- .../iq/dataverse/util/json/JsonParser.java | 16 +++++ .../harvard/iq/dataverse/api/DatasetsIT.java | 49 +++++++++++++-- .../edu/harvard/iq/dataverse/api/UtilIT.java | 29 +++++++++ .../UpdateDatasetTermsOfUseCommandTest.java | 55 +++++++++++++++++ .../json/update-dataset-terms-access.json | 20 +++++++ 9 files changed, 278 insertions(+), 6 deletions(-) create mode 100644 scripts/search/tests/data/update-dataset-terms.json create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommand.java create mode 100644 src/test/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommandTest.java create mode 100644 src/test/resources/json/update-dataset-terms-access.json diff --git a/scripts/search/tests/data/update-dataset-terms.json b/scripts/search/tests/data/update-dataset-terms.json new file mode 100644 index 00000000000..dbe6380ca56 --- /dev/null +++ b/scripts/search/tests/data/update-dataset-terms.json @@ -0,0 +1,20 @@ +{ + "termsOfAccess": { + "fileAccessRequest": "true", + "termsOfAccessForRestrictedFiles": "For access to restriced files please see read me file", + "dataAccessPlace": "dataAccessPlace", + "originalArchive": "originalArchive", + "availabilityStatus": "availabilityStatus", + "contactForAccess": "contactForAccess", + "sizeOfCollection": "sizeOfCollection", + "studyCompletion": "studyCompletion", + "termsOfUse": "termsOfUse", + "confidentialityDeclaration": "confidentialityDeclaration", + "specialPermissions": "specialPermissions", + "restrictions": "restrictions", + "citationRequirements": "citationRequirements", + "depositorRequirements": "depositorRequirements", + "conditions": "conditions", + "disclaimer": "disclaimer" + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index fb97c761ef1..355db7c8c60 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1152,6 +1152,39 @@ public Response editVersionMetadata(@Context ContainerRequestContext crc, String return ex.getResponse(); } } + + @PUT + @AuthRequired + @Path("{id}/editTerms") + public Response editVersionTermsOfUse(@Context ContainerRequestContext crc, String jsonBody, @PathParam("id") String id, + @QueryParam("sourceLastUpdateTime") String sourceLastUpdateTime) { + try { + Dataset dataset = findDatasetOrDie(id); + + if (sourceLastUpdateTime != null) { + validateInternalTimestampIsNotOutdated(dataset, sourceLastUpdateTime); + } + + JsonObject json = JsonUtil.getJsonObject(jsonBody); + System.out.print(jsonBody); + TermsOfUseAndAccess toua = jsonParser().parseTermsOfUseAndAccess(json); + + + toua.setDatasetVersion(dataset.getLatestVersion()); + + + DatasetVersion updatedVersion = execCommand(new UpdateDatasetTermsOfUseCommand(dataset, toua, createDataverseRequest(getRequestUser(crc)))).getLatestVersion(); + + return ok(json(updatedVersion, true)); + + } catch (JsonParseException ex) { + logger.log(Level.SEVERE, "Semantic error parsing dataset terms update Json: " + ex.getMessage(), ex); + return error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("datasets.api.editMetadata.error.parseUpdate", List.of(ex.getMessage()))); + } catch (WrappedResponse ex) { + logger.log(Level.SEVERE, "Update terms of use error: " + ex.getMessage(), ex); + return ex.getResponse(); + } + } /** * @deprecated This was shipped as a GET but should have been a POST, see https://github.com/IQSS/dataverse/issues/2431 diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommand.java new file mode 100644 index 00000000000..6e7a5c29e57 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommand.java @@ -0,0 +1,59 @@ +/* + * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license + * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template + */ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.TermsOfUseAndAccess; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.InvalidCommandArgumentsException; +import edu.harvard.iq.dataverse.util.BundleUtil; + +/** + * + * @author stephenkraffmiller + */ +@RequiredPermissions(Permission.EditDataset) +public class UpdateDatasetTermsOfUseCommand extends AbstractDatasetCommand{ + + + private final Dataset dataset; + private final TermsOfUseAndAccess termsOfUseAndAccess; + private final UpdateDatasetVersionCommand updateDatasetVersionCommand; + + public UpdateDatasetTermsOfUseCommand(Dataset dataset, TermsOfUseAndAccess termsOfUseAndAccess, DataverseRequest request) { + this(dataset, termsOfUseAndAccess, request, null); + } + + //Command included for testing purposes + public UpdateDatasetTermsOfUseCommand( Dataset dataset, TermsOfUseAndAccess termsOfUseAndAccess, DataverseRequest aRequest, UpdateDatasetVersionCommand updateDatasetVersionCommand ) { + super(aRequest, dataset); + this.dataset = dataset; + this.termsOfUseAndAccess = termsOfUseAndAccess; + this.updateDatasetVersionCommand = updateDatasetVersionCommand; + } + + @Override + public Dataset execute(CommandContext ctxt) throws CommandException { + DatasetVersion datasetVersion = dataset.getOrCreateEditVersion(); + + + datasetVersion.setTermsOfUseAndAccess(termsOfUseAndAccess); + + datasetVersion.setVersionState(DatasetVersion.VersionState.DRAFT); + termsOfUseAndAccess.setDatasetVersion(datasetVersion); + System.out.print("From input...."); + System.out.print(termsOfUseAndAccess.getConfidentialityDeclaration()); + System.out.print("From DatasetVersion...."); + System.out.print(datasetVersion.getTermsOfUseAndAccess().getConfidentialityDeclaration()); + System.out.print("$$$$$$$$$$$$$$$$$$$$$$$$"); + return ctxt.engine().submit(updateDatasetVersionCommand == null ? new UpdateDatasetVersionCommand(this.dataset, getRequest()) : updateDatasetVersionCommand); + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java index 209791faafb..1a3a55be4a0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java @@ -100,7 +100,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { if ( ! (getUser() instanceof AuthenticatedUser) ) { throw new IllegalCommandException("Only authenticated users can update datasets", this); } - + Dataset theDataset = getDataset(); ctxt.permissions().checkUpdateDatasetVersionLock(theDataset, getRequest(), this); Dataset savedDataset = null; @@ -136,6 +136,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { } getDataset().getOrCreateEditVersion(fmVarMet).setDatasetFields(getDataset().getOrCreateEditVersion(fmVarMet).initDatasetFields()); + validateOrDie(getDataset().getOrCreateEditVersion(fmVarMet), isValidateLenient()); final DatasetVersion editVersion = getDataset().getOrCreateEditVersion(fmVarMet); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index bd97bcf4274..fd662b975a6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -377,6 +377,22 @@ public static > List parseEnumsFromArray(JsonArray enumsArr } return enums; } + + public TermsOfUseAndAccess parseTermsOfUseAndAccess(JsonObject obj) throws JsonParseException { + + TermsOfUseAndAccess terms = new TermsOfUseAndAccess(); + + terms.setTermsOfUse(obj.getString("termsOfUse", null)); + terms.setConfidentialityDeclaration(obj.getString("confidentialityDeclaration", null)); + terms.setSpecialPermissions(obj.getString("specialPermissions", null)); + terms.setRestrictions(obj.getString("restrictions", null)); + terms.setCitationRequirements(obj.getString("citationRequirements", null)); + terms.setDepositorRequirements(obj.getString("depositorRequirements", null)); + terms.setConditions(obj.getString("conditions", null)); + terms.setDisclaimer(obj.getString("disclaimer", null)); + + return terms; + } public DatasetVersion parseDatasetVersion(JsonObject obj) throws JsonParseException { return parseDatasetVersion(obj, new DatasetVersion()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index b273502e6ed..91196b6b9d5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -3969,8 +3969,8 @@ private JsonObject getDataAsJsonObject(String body) { private String getData(String body) { return getDataAsJsonObject(body).toString(); } - - @Test + + @Test public void testFilesUnchangedAfterDatasetMetadataUpdate() throws IOException { Response createUser = UtilIT.createRandomUser(); createUser.prettyPrint(); @@ -4016,20 +4016,59 @@ public void testFilesUnchangedAfterDatasetMetadataUpdate() throws IOException { .statusCode(OK.getStatusCode()) .body("data.latestVersion.files[0].label", equalTo("run.sh")) .body("data.latestVersion.files[0].directoryLabel", equalTo("code")); - + String pathToJsonFile = "doc/sphinx-guides/source/_static/api/dataset-update-metadata.json"; Response updateTitle = UtilIT.updateDatasetMetadataViaNative(datasetPid, pathToJsonFile, apiToken); updateTitle.prettyPrint(); updateTitle.then().assertThat() .statusCode(OK.getStatusCode()); - + Response getDatasetJsonAfterUpdate = UtilIT.nativeGet(datasetId, apiToken); getDatasetJsonAfterUpdate.prettyPrint(); getDatasetJsonAfterUpdate.then().assertThat() .statusCode(OK.getStatusCode()) .body("data.latestVersion.files[0].label", equalTo("run.sh")) .body("data.latestVersion.files[0].directoryLabel", equalTo("code")); - + + } + + @Test + public void testUpdateDatasetTerms() throws IOException { + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + createUser.then().assertThat() + .statusCode(OK.getStatusCode()); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverse = UtilIT.createRandomDataverse(apiToken); + createDataverse.prettyPrint(); + createDataverse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + String datasetPid = JsonPath.from(createDataset.asString()).getString("data.persistentId"); + Integer datasetId = JsonPath.from(createDataset.asString()).getInt("data.id"); + + UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken).then().assertThat().statusCode(OK.getStatusCode()); + UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken).then().assertThat().statusCode(OK.getStatusCode()); + + Response getDatasetJsonBeforeUpdate = UtilIT.nativeGet(datasetId, apiToken); + getDatasetJsonBeforeUpdate.prettyPrint(); + getDatasetJsonBeforeUpdate.then().assertThat() + .statusCode(OK.getStatusCode()); + + String pathToJsonFile = "src/test/resources/json/update-dataset-terms-access.json"; + Response updateTerms = UtilIT.updateDatasetTermsAndAccess(datasetPid, apiToken, pathToJsonFile); + updateTerms.prettyPrint(); + updateTerms.then().assertThat() + .statusCode(OK.getStatusCode()); + } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index eba8181e566..69096a52a5b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -3859,6 +3859,35 @@ static Response updateDatasetJsonLDMetadata(Integer datasetId, String apiToken, .put("/api/datasets/" + datasetId + "/metadata?replace=" + replace); return response; } + + static Response updateDatasetTermsAndAccess(String idOrPersistentIdOfDataset, String apiToken, String pathToJsonFile) { + String idInPath = idOrPersistentIdOfDataset; + String optionalQueryParam = ""; + if (!NumberUtils.isCreatable(idOrPersistentIdOfDataset)) { + idInPath = ":persistentId"; + optionalQueryParam = "?persistentId=" + idOrPersistentIdOfDataset; + } + + String jsonIn = getDatasetTerms(pathToJsonFile); + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .contentType("application/json") + .body(jsonIn) + .put("/api/datasets/" + idInPath + "/editTerms" + optionalQueryParam); + return response; + } + + public static String getDatasetTerms(String pathToJsonFile) { + File datasetTermsJson = new File(pathToJsonFile); + try { + String datasetTermsAsJson = new String(Files.readAllBytes(Paths.get(datasetTermsJson.getAbsolutePath()))); + return datasetTermsAsJson; + } catch (IOException ex) { + Logger.getLogger(UtilIT.class.getName()).log(Level.SEVERE, null, ex); + return null; + } + + } static Response deleteDatasetJsonLDMetadata(Integer datasetId, String apiToken, String jsonLDBody) { Response response = given() diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommandTest.java new file mode 100644 index 00000000000..e9d357ecbe6 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommandTest.java @@ -0,0 +1,55 @@ +/* + * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license + * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template + */ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.TermsOfUseAndAccess; +import edu.harvard.iq.dataverse.dataset.DatasetFieldsValidator; +import edu.harvard.iq.dataverse.engine.DataverseEngine; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import org.junit.jupiter.api.BeforeEach; +import org.mockito.Mock; +import static org.mockito.Mockito.when; +import org.mockito.MockitoAnnotations; + +/** + * + * @author stephenkraffmiller + */ +public class UpdateDatasetTermsOfUseCommandTest { + @Mock + private UpdateDatasetVersionCommand updateDatasetVersionCommandStub; + @Mock + private DataverseEngine dataverseEngineMock; + @Mock + private Dataset datasetMock; + @Mock + private DataverseRequest dataverseRequestStub; + @Mock + private TermsOfUseAndAccess termsOfUseAndAccessStub; + @Mock + private CommandContext commandContextMock; + @Mock + private DatasetVersion datasetVersionMock; + @Mock + private DatasetFieldsValidator datasetFieldsValidatorMock; + + private UpdateDatasetFieldsCommand sut; + + @BeforeEach + public void setUp() throws CommandException { + MockitoAnnotations.openMocks(this); + + when(dataverseEngineMock.submit(updateDatasetVersionCommandStub)).thenReturn(datasetMock); + when(commandContextMock.engine()).thenReturn(dataverseEngineMock); + when(commandContextMock.datasetFieldsValidator()).thenReturn(datasetFieldsValidatorMock); + when(datasetMock.getOrCreateEditVersion()).thenReturn(datasetVersionMock); + when(datasetVersionMock.getTermsOfUseAndAccess()).thenReturn(termsOfUseAndAccessStub); + } + +} diff --git a/src/test/resources/json/update-dataset-terms-access.json b/src/test/resources/json/update-dataset-terms-access.json new file mode 100644 index 00000000000..dbe6380ca56 --- /dev/null +++ b/src/test/resources/json/update-dataset-terms-access.json @@ -0,0 +1,20 @@ +{ + "termsOfAccess": { + "fileAccessRequest": "true", + "termsOfAccessForRestrictedFiles": "For access to restriced files please see read me file", + "dataAccessPlace": "dataAccessPlace", + "originalArchive": "originalArchive", + "availabilityStatus": "availabilityStatus", + "contactForAccess": "contactForAccess", + "sizeOfCollection": "sizeOfCollection", + "studyCompletion": "studyCompletion", + "termsOfUse": "termsOfUse", + "confidentialityDeclaration": "confidentialityDeclaration", + "specialPermissions": "specialPermissions", + "restrictions": "restrictions", + "citationRequirements": "citationRequirements", + "depositorRequirements": "depositorRequirements", + "conditions": "conditions", + "disclaimer": "disclaimer" + } +} From 9fff5df5d3f198a8c2d3ea9f980a5e780a9cec96 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 25 Sep 2025 13:56:04 -0400 Subject: [PATCH 295/634] #11772 fix parser --- .../iq/dataverse/util/json/JsonParser.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index fd662b975a6..b74f852202c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -379,19 +379,19 @@ public static > List parseEnumsFromArray(JsonArray enumsArr } public TermsOfUseAndAccess parseTermsOfUseAndAccess(JsonObject obj) throws JsonParseException { - - TermsOfUseAndAccess terms = new TermsOfUseAndAccess(); - - terms.setTermsOfUse(obj.getString("termsOfUse", null)); - terms.setConfidentialityDeclaration(obj.getString("confidentialityDeclaration", null)); - terms.setSpecialPermissions(obj.getString("specialPermissions", null)); - terms.setRestrictions(obj.getString("restrictions", null)); - terms.setCitationRequirements(obj.getString("citationRequirements", null)); - terms.setDepositorRequirements(obj.getString("depositorRequirements", null)); - terms.setConditions(obj.getString("conditions", null)); - terms.setDisclaimer(obj.getString("disclaimer", null)); - - return terms; + JsonObject terms = obj.getJsonObject("termsOfAccess"); + TermsOfUseAndAccess toaa = new TermsOfUseAndAccess(); + + toaa.setTermsOfUse(terms.getString("termsOfUse", null)); + toaa.setConfidentialityDeclaration(terms.getString("confidentialityDeclaration", null)); + toaa.setSpecialPermissions(terms.getString("specialPermissions", null)); + toaa.setRestrictions(terms.getString("restrictions", null)); + toaa.setCitationRequirements(terms.getString("citationRequirements", null)); + toaa.setDepositorRequirements(terms.getString("depositorRequirements", null)); + toaa.setConditions(terms.getString("conditions", null)); + toaa.setDisclaimer(terms.getString("disclaimer", null)); + + return toaa; } public DatasetVersion parseDatasetVersion(JsonObject obj) throws JsonParseException { From 892193c5e76f373c08b953a5f516fa257e84af1a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 18 Sep 2025 14:16:23 -0400 Subject: [PATCH 296/634] get `make docs-html` working #11829 --- makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/makefile b/makefile index 315ff9c508c..1ffdb627275 100644 --- a/makefile +++ b/makefile @@ -1,5 +1,5 @@ - -SPHINX_VERSION = $(shell grep "Sphinx" ./doc/sphinx-guides/requirements.txt | awk -F'==' '{print $$2}') +# We use "Sphinx=" to avoid packages like Sphinx-Substitution-Extensions +SPHINX_VERSION = $(shell grep "Sphinx=" ./doc/sphinx-guides/requirements.txt | awk -F'==' '{print $$2}') docs-html: docker run -it --rm -v $$(pwd):/docs sphinxdoc/sphinx:$(SPHINX_VERSION) bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make clean && make html" @@ -11,4 +11,4 @@ docs-epub: docs-all: docker run -it --rm -v $$(pwd):/docs sphinxdoc/sphinx:$(SPHINX_VERSION) bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make clean && make html && make epub" - docker run -it --rm -v $$(pwd):/docs sphinxdoc/sphinx-latexpdf:$(SPHINX_VERSION) bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make latexpdf LATEXMKOPTS=\"-interaction=nonstopmode\"; cd ../.. && ls -1 doc/sphinx-guides/build/latex/Dataverse.pdf" \ No newline at end of file + docker run -it --rm -v $$(pwd):/docs sphinxdoc/sphinx-latexpdf:$(SPHINX_VERSION) bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make latexpdf LATEXMKOPTS=\"-interaction=nonstopmode\"; cd ../.. && ls -1 doc/sphinx-guides/build/latex/Dataverse.pdf" From 92d36c61bc7ffd6675a796af98d582e0bfa8b312 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 18 Sep 2025 15:20:03 -0400 Subject: [PATCH 297/634] suggest Docker first, require Python 3.10, reorg --- .../source/contributor/documentation.md | 75 ++++++++++--------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/documentation.md b/doc/sphinx-guides/source/contributor/documentation.md index 2a8d6794921..2585a9f0712 100644 --- a/doc/sphinx-guides/source/contributor/documentation.md +++ b/doc/sphinx-guides/source/contributor/documentation.md @@ -50,76 +50,77 @@ If you would like to read more about the Dataverse's use of GitHub, please see t ## Building the Guides with Sphinx -While the "quick fix" technique shown above should work fine for minor changes, especially for larger changes, we recommend installing Sphinx on your computer or using a Sphinx Docker container to build the guides locally so you can get an accurate preview of your changes. +While the "quick fix" technique shown above should work fine for minor changes, in many cases, you're going to want to preview changes locally before committing them. -In case you decide to use a Sphinx Docker container to build the guides, you can skip the next two installation sections, but you will need to have Docker installed. +Before we worry about pushing changes to the code, let's make sure we can build the guides. -### Installing Sphinx +Go to and click "Code" and then follow the instructions to clone the code locally. -First, make a fork of and clone your fork locally. Then change to the ``doc/sphinx-guides`` directory. +### Makefile and Docker -``cd doc/sphinx-guides`` +From a terminal, switch to the "dataverse" directory you just cloned. -Create a Python virtual environment, activate it, then install dependencies: - -``python3 -m venv venv`` +`cd dataverse` -``source venv/bin/activate`` +Then try building the HTML version of the guides. -``pip install -r requirements.txt`` +`make docs-html` -### Installing GraphViz +If all goes well, you should be able open `doc/sphinx-guides/build/html/index.html` to see the guides you just built. -In some parts of the documentation, graphs are rendered as images using the Sphinx GraphViz extension. +Here is the complete list of commands. -Building the guides requires the ``dot`` executable from GraphViz. +- `make docs-html` +- `make docs-pdf` +- `make docs-epub` +- `make docs-all` -This requires having [GraphViz](https://graphviz.org) installed and either having ``dot`` on the path or -[adding options to the `make` call](https://groups.google.com/forum/#!topic/sphinx-users/yXgNey_0M3I). +### Docker -On a Mac we recommend installing GraphViz through [Homebrew](). Once you have Homebrew installed and configured to work with your shell, you can type `brew install graphviz`. +Also from the "dataverse" directory (the root of the git repo), you can try this command: -### Editing and Building the Guides +`docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx:7.2.6 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make html"` -To edit the existing documentation: +### Sphinx Installed Locally -- Create a branch (see {ref}`how-to-make-a-pull-request`). -- In ``doc/sphinx-guides/source`` you will find the .rst files that correspond to https://guides.dataverse.org. -- Using your preferred text editor, open and edit the necessary files, or create new ones. +First, run `python --version` or `python3 --version` to determine the version of Python you have. If you don't have Python 3.10 or higher, you must upgrade. -Once you are done, you can preview the changes by building the guides locally. As explained, you can build the guides with Sphinx locally installed, or with a Docker container. +Next, change to the `doc/sphinx-guides` directory. -#### Building the Guides with Sphinx Installed Locally +`cd doc/sphinx-guides` -Open a terminal, change directories to `doc/sphinx-guides`, activate (or reactivate) your Python virtual environment, and build the guides. +Create a Python virtual environment, activate it, then install dependencies: -`cd doc/sphinx-guides` +`python3 -m venv venv` `source venv/bin/activate` -`make clean` +`pip install -r requirements.txt` -`make html` +Next, install [GraphViz](https://graphviz.org) because building the guides requires having the `dot` executable from GraphViz either on the path or passed [as an argument](https://groups.google.com/g/sphinx-users/c/yXgNey_0M3I/m/3T2NipFlBgAJ). -#### Building the Guides with a Sphinx Docker Container and a Makefile +On a Mac we recommend installing GraphViz through [Homebrew](). Once you have Homebrew installed and configured to work with your shell, you can type `brew install graphviz`. -We have added a Makefile to simplify the process of building the guides using a Docker container, you can use the following commands from the repository root: +Finally, you can try building the guides with the following command. -- `make docs-html` -- `make docs-pdf` -- `make docs-epub` -- `make docs-all` +`make html` -#### Building the Guides with a Sphinx Docker Container and CLI +If all goes well, you should be able open `doc/sphinx-guides/build/html/index.html` to see the guides you just built. -If you want to build the guides using a Docker container, execute the following command in the repository root: +## Editing, Building, and Previewing the Guides -`docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx:7.2.6 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make html"` +To edit the existing documentation: + +- Create a branch (see {ref}`how-to-make-a-pull-request`). +- In `doc/sphinx-guides/source` you will find the .rst or .md files that correspond to https://guides.dataverse.org. +- Using your preferred text editor, open and edit the necessary files, or create new ones. -#### Previewing the Guides +Once you are done, you can preview the changes by building the guides using one of the options above. After Sphinx is done processing the files you should notice that the `html` folder in `doc/sphinx-guides/build` directory has been updated. You can click on the files in the `html` folder to preview the changes. +## Making a Pull Request + Now you can make a commit with the changes to your own fork in GitHub and submit a pull request. See {ref}`how-to-make-a-pull-request`. ## Writing Guidelines From 3ea1ef24ff722baacc92f4abefab484e441149ec Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 18 Sep 2025 15:25:16 -0400 Subject: [PATCH 298/634] custom titles in .md --- doc/sphinx-guides/source/contributor/documentation.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/documentation.md b/doc/sphinx-guides/source/contributor/documentation.md index 2585a9f0712..2bdcbebb340 100644 --- a/doc/sphinx-guides/source/contributor/documentation.md +++ b/doc/sphinx-guides/source/contributor/documentation.md @@ -154,16 +154,22 @@ If the page is written in Markdown (.md), use this form: ### Links -Getting links right with .rst files can be tricky. +Getting links right can be tricky. #### Custom Titles -You can use a custom title when linking to a document like this: +In .rst files you can use a custom title when linking to a document like this: :doc:`Custom title ` See also +In .md files, the same pattern can be used. Here's an example of using a custom title with a ref: + + {ref}`Log in ` + +See also + ### Images A good documentation is just like a website enhanced and upgraded by adding high quality and self-explanatory images. Often images depict a lot of written text in a simple manner. Within our Sphinx docs, you can add them in two ways: a) add a PNG image directly and include or b) use inline description languages like GraphViz (current only option). From 84585e12c45feceb1899bdd6a35bb4dfa279e173 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 26 Sep 2025 10:05:32 -0400 Subject: [PATCH 299/634] try docker before make command that wraps it --- .../source/contributor/documentation.md | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/doc/sphinx-guides/source/contributor/documentation.md b/doc/sphinx-guides/source/contributor/documentation.md index 2bdcbebb340..7f240d15790 100644 --- a/doc/sphinx-guides/source/contributor/documentation.md +++ b/doc/sphinx-guides/source/contributor/documentation.md @@ -56,30 +56,31 @@ Before we worry about pushing changes to the code, let's make sure we can build Go to and click "Code" and then follow the instructions to clone the code locally. -### Makefile and Docker +### Docker + +Install [Docker Desktop](https://www.docker.com/products/docker-desktop/). -From a terminal, switch to the "dataverse" directory you just cloned. +From a terminal, switch to the "dataverse" directory you just cloned. This is the root of the git repo. `cd dataverse` -Then try building the HTML version of the guides. +Then try running this command: -`make docs-html` +`docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx:7.2.6 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make html"` If all goes well, you should be able open `doc/sphinx-guides/build/html/index.html` to see the guides you just built. -Here is the complete list of commands. +#### Docker with a Makefile -- `make docs-html` -- `make docs-pdf` -- `make docs-epub` -- `make docs-all` +Once you've confirmed you have Docker working, if you have [make](https://en.wikipedia.org/wiki/Make_(software)) installed, you can try the following commands: -### Docker +`make docs-html` -Also from the "dataverse" directory (the root of the git repo), you can try this command: +`make docs-pdf` -`docker run -it --rm -v $(pwd):/docs sphinxdoc/sphinx:7.2.6 bash -c "cd doc/sphinx-guides && pip3 install -r requirements.txt && make html"` +`make docs-epub` + +`make docs-all` ### Sphinx Installed Locally From 9b3c06579a82c05ba83820229b30f0b9a8b6e348 Mon Sep 17 00:00:00 2001 From: Maham Maham <116267645+Maham802@users.noreply.github.com> Date: Fri, 26 Sep 2025 18:31:05 +0200 Subject: [PATCH 300/634] docs: update dataverse-ansible link --- doc/sphinx-guides/source/admin/monitoring.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/admin/monitoring.rst b/doc/sphinx-guides/source/admin/monitoring.rst index 16bb18b7ad2..ef4e4f4f206 100644 --- a/doc/sphinx-guides/source/admin/monitoring.rst +++ b/doc/sphinx-guides/source/admin/monitoring.rst @@ -149,7 +149,7 @@ Tips: - Use **Enhanced Monitoring**. Enhanced Monitoring gathers its metrics from an agent on the instance. See `Enhanced Monitoring docs `_. - It's possible to view and act on **RDS Events** such as snapshots, parameter changes, etc. See `Working with Amazon RDS events `_ for details. - RDS monitoring is available via API and the ``aws`` command line tool. For example, see `Retrieving metrics with the Performance Insights API `_. -- To play with monitoring RDS using a server configured by `dataverse-ansible `_ set ``use_rds`` to true to skip some steps that aren't necessary when using RDS. See also the :doc:`/developers/deployment` section of the Developer Guide. +- To play with monitoring RDS using a server configured by `dataverse-ansible `_ set ``use_rds`` to true to skip some steps that aren't necessary when using RDS. See also the :doc:`/developers/deployment` section of the Developer Guide. MicroProfile Metrics endpoint ----------------------------- From ee21a115c667def8bf21f58765816b9aba9be04b Mon Sep 17 00:00:00 2001 From: Maham Maham <116267645+Maham802@users.noreply.github.com> Date: Fri, 26 Sep 2025 18:58:14 +0200 Subject: [PATCH 301/634] docs: update dataverse-ansible link --- doc/sphinx-guides/source/installation/prep.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/prep.rst b/doc/sphinx-guides/source/installation/prep.rst index abb4349d3ad..33b1aa3c7b5 100644 --- a/doc/sphinx-guides/source/installation/prep.rst +++ b/doc/sphinx-guides/source/installation/prep.rst @@ -26,7 +26,7 @@ Advanced Installation There are some community-lead projects to use configuration management tools such as Ansible and Puppet to automate the installation and configuration of the Dataverse Software, but support for these solutions is limited to what the Dataverse Community can offer as described in each project's webpage: -- https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible +- https://github.com/gdcc/dataverse-ansible - https://gitlab.com/lip-computing/dataverse - https://github.com/IQSS/dataverse-puppet From 05c8dae0ed58fa6e19087fd53fd030792500fc2e Mon Sep 17 00:00:00 2001 From: Maham Maham <116267645+Maham802@users.noreply.github.com> Date: Fri, 26 Sep 2025 19:05:08 +0200 Subject: [PATCH 302/634] docs: update dataverse-ansible libk --- doc/sphinx-guides/source/api/external-tools.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst index ae0e44b36aa..389519318db 100644 --- a/doc/sphinx-guides/source/api/external-tools.rst +++ b/doc/sphinx-guides/source/api/external-tools.rst @@ -202,7 +202,7 @@ Testing Your External Tool As the author of an external tool, you are not expected to learn how to install and operate a Dataverse installation. There's a very good chance your tool can be added to a server Dataverse Community developers use for testing if you reach out on any of the channels listed under :ref:`getting-help-developers` in the Developer Guide. -By all means, if you'd like to install a Dataverse installation yourself, a number of developer-centric options are available. For example, there's a script to spin up a Dataverse installation on EC2 at https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible . The process for using curl to add your external tool to your Dataverse installation is documented under :ref:`managing-external-tools` in the Admin Guide. +By all means, if you'd like to install a Dataverse installation yourself, a number of developer-centric options are available. For example, there's a script to spin up a Dataverse installation on EC2 at https://github.com/gdcc/dataverse-ansible . The process for using curl to add your external tool to your Dataverse installation is documented under :ref:`managing-external-tools` in the Admin Guide. Spreading the Word About Your External Tool ------------------------------------------- @@ -219,7 +219,7 @@ If you've thought to yourself that there ought to be an app store for Dataverse Demoing Your External Tool ++++++++++++++++++++++++++ -https://demo.dataverse.org is the place to play around with the Dataverse Software and your tool can be included. Please email support@dataverse.org to start the conversation about adding your tool. Additionally, you are welcome to open an issue at https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible which already includes a number of the tools listed above. +https://demo.dataverse.org is the place to play around with the Dataverse Software and your tool can be included. Please email support@dataverse.org to start the conversation about adding your tool. Additionally, you are welcome to open an issue at https://github.com/gdcc/dataverse-ansible which already includes a number of the tools listed above. Announcing Your External Tool +++++++++++++++++++++++++++++ From d6616b99d83b2c7efd02627515ce64a0d562501a Mon Sep 17 00:00:00 2001 From: Maham Maham <116267645+Maham802@users.noreply.github.com> Date: Fri, 26 Sep 2025 19:11:38 +0200 Subject: [PATCH 303/634] docs: update dataverse-ansible link --- doc/sphinx-guides/source/developers/testing.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index 1690864d453..8f9ce645a2e 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -169,9 +169,9 @@ different people. For our purposes, an integration test can have two flavors: Running the Full API Test Suite Using EC2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -**Prerequisite:** To run the API test suite in an EC2 instance you should first follow the steps in the :doc:`deployment` section to get set up with the AWS binary to launch EC2 instances. If you're here because you just want to spin up a branch, you'll still want to follow the AWS deployment setup steps, but may find the `ec2-create README.md `_ Quick Start section helpful. +**Prerequisite:** To run the API test suite in an EC2 instance you should first follow the steps in the :doc:`deployment` section to get set up with the AWS binary to launch EC2 instances. If you're here because you just want to spin up a branch, you'll still want to follow the AWS deployment setup steps, but may find the `ec2-create README.md `_ Quick Start section helpful. -You may always retrieve a current copy of the ec2-create-instance.sh script and accompanying group_var.yml file from the `dataverse-ansible repo `_. Since we want to run the test suite, let's grab the group_vars used by Jenkins: +You may always retrieve a current copy of the ec2-create-instance.sh script and accompanying group_var.yml file from the `dataverse-ansible repo `_. Since we want to run the test suite, let's grab the group_vars used by Jenkins: - `ec2-create-instance.sh `_ - `jenkins.yml `_ @@ -184,7 +184,7 @@ Edit ``jenkins.yml`` to set the desired GitHub repo and branch, and to adjust an - ``dataverse.unittests.enabled: true`` - ``dataverse.sampledata.enabled: true`` -If you wish, you may pass the script a ``-l`` flag with a local relative path in which the script will `copy various logs `_ at the end of the test suite for your review. +If you wish, you may pass the script a ``-l`` flag with a local relative path in which the script will `copy various logs `_ at the end of the test suite for your review. Finally, run the script: @@ -526,7 +526,7 @@ Browser-Based Testing Installation Testing ~~~~~~~~~~~~~~~~~~~~ -- Work with @donsizemore to automate testing of https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible +- Work with @donsizemore to automate testing of https://github.com/gdcc/dataverse-ansible Future Work on Load/Performance Testing ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -538,4 +538,4 @@ Future Work on Load/Performance Testing Future Work on Accessibility Testing ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Using https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible and hooks available from accessibility testing tools, automate the running of accessibility tools on PRs so that developers will receive quicker feedback on proposed code changes that reduce the accessibility of the application. +- Using https://github.com/gdcc/dataverse-ansible and hooks available from accessibility testing tools, automate the running of accessibility tools on PRs so that developers will receive quicker feedback on proposed code changes that reduce the accessibility of the application. From a8e1a6c121edad8531bbfe168c5a83b82a2c0ac3 Mon Sep 17 00:00:00 2001 From: Maham Maham <116267645+Maham802@users.noreply.github.com> Date: Fri, 26 Sep 2025 19:26:39 +0200 Subject: [PATCH 304/634] docs: update dataverse-ansible link --- doc/sphinx-guides/source/developers/deployment.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/developers/deployment.rst b/doc/sphinx-guides/source/developers/deployment.rst index 46cf95dae54..d3e0c33a8f2 100755 --- a/doc/sphinx-guides/source/developers/deployment.rst +++ b/doc/sphinx-guides/source/developers/deployment.rst @@ -78,7 +78,7 @@ Amazon offers instructions on using an IAM role to grant permissions to applicat Configure Ansible File (Optional) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In order to configure Dataverse installation settings such as the password of the dataverseAdmin user, download https://raw.githubusercontent.com/GlobalDataverseCommunityConsortium/dataverse-ansible/master/defaults/main.yml and edit the file to your liking. +In order to configure Dataverse installation settings such as the password of the dataverseAdmin user, download https://raw.githubusercontent.com/gdcc/dataverse-ansible/master/defaults/main.yml and edit the file to your liking. You can skip this step if you're fine with the values in the "main.yml" file in the link above. @@ -89,7 +89,7 @@ Once you have done the configuration above, you are ready to try running the "ec Download `ec2-create-instance.sh`_ and put it somewhere reasonable. For the purpose of these instructions we'll assume it's in the "Downloads" directory in your home directory. -.. _ec2-create-instance.sh: https://raw.githubusercontent.com/GlobalDataverseCommunityConsortium/dataverse-ansible/master/ec2/ec2-create-instance.sh +.. _ec2-create-instance.sh: https://raw.githubusercontent.com/gdcc/dataverse-ansible/master/ec2/ec2-create-instance.sh To run the script, you can make it executable (``chmod 755 ec2-create-instance.sh``) or run it with bash, like this with ``-h`` as an argument to print the help: From 67b0fcbe01801739efd7cf37666b241012c92c03 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 26 Sep 2025 14:45:50 -0400 Subject: [PATCH 305/634] add a big data admin guide --- .../source/admin/big-data-administration.rst | 305 ++++++++++++++++++ doc/sphinx-guides/source/admin/index.rst | 1 + 2 files changed, 306 insertions(+) create mode 100644 doc/sphinx-guides/source/admin/big-data-administration.rst diff --git a/doc/sphinx-guides/source/admin/big-data-administration.rst b/doc/sphinx-guides/source/admin/big-data-administration.rst new file mode 100644 index 00000000000..dba872aed5e --- /dev/null +++ b/doc/sphinx-guides/source/admin/big-data-administration.rst @@ -0,0 +1,305 @@ +Scaling Dataverse with Data Size +================================ + +This guide is intended to help administrators configure Dataverse appropriately to handle the amount of data their installation needs to manage. + +Scaling is a complex subject: there are many options available in Dataverse that can improve performance with larger scale data, some of which +work differently than Dataverse's default configuration, potentially requiring user education, and some which can require additional expertise to manage. + +In general, there are three dimensions in which Dataverse can scale: + +1. **Storage size of individual files and aggregate storage size** +2. **Number of Files per Dataset** +3. **Number of Datasets** + + +.. contents:: |toctitle| + :local: + +Storage: Choosing the Right Store +--------------------------------- + +The main issues for handling larger files and larger aggregate data size relate to the performance of the storage used and how involved the Dataverse server is in the data transfer. +With appropriate configuration, Dataverse can support file sizes and aggregate dataset sizes into the terabyte scale and beyond. + +The primary choice in Dataverse related to storage is which type(s) of "store" (also called "Storage driver") to use: + +File Stores +~~~~~~~~~~~ + +The default storage option in Dataverse is based on a local file system. When files are transferred to Dataverse, they are first stored in a +temporary location on the Dataverse server. Any zip files uploaded are unzipped to create multiple individual file entries. Once an upload is completed, +Dataverse copies the files to permanent storage. Dataverse also takes advantage of the file being local to inspect it's bytes to determine it's +MIME type, and, for tabular data, to 'ingest' it - extracting metadata about the variables used in the file and creating a tab-separated value (TSV) +version of the file. + +Benefits: + +- This option requires no external services and can potentially handle files into the gigabyte (GB) size range. For smaller institutions, +and in disciplines where datasets do not have more than a few hundred files and files are not too large, this can be the simplest option. +- Unzipping of zip archives can be simpler for users than having to upload many individual files and was, at one time, the only way to +preserve file path names when uploading. + +Challenges: In general file storage is not a good option for larger data sizes - both in terms of file size and number of files. Contributing factors include: + +- Because temporary storage is used, transfers will temporarily use several times as much space as the final transfer. Unzipping also increases the final storage size of a dataset. +- Because all uploads use the same temporary storage, temporary storage must be large enough to handle multiple users uploading data. +- Each file is uploaded as a single HTTP request, which can cause long transfer times which, in turn, can trigger timeout errors in Dataverse or any proxy or load balancer in front of Dataverse. +- Uploading many files at once can trigger any rate limiter in front of the Dataverse server (i.e. used to throttle use by AI) resulting in failures +- Because transfers (both uploads and downloads) are handled by the Dataverse server, they add to server processing load which can affect overall performance. +- Cost: local file storage must be provisioned in advance based on anticipated demand. It can involve up-front costs (for a local disk), or, when procured from a + cloud provider, is likely to be more expensive than object storage from that provider (see below) + + +S3 Stores: Object Storage via S3 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A more scalable option for storage is to use an object store with transfers managed using the Simple Storage Service (S3) protocol. S3-compatible storage can +easily be bought (rented) from major cloud providers, but may also be available from institutional clouds. It is also possible to run open-source software to provide +S3 storage over a local file system (making it possible to enjoy the advantages discussed below while still leveraging local file storage). + +While S3 Stores can be configured to handle uploads and downloads as with file storage (with zip files being unzipped, but having many of the same challenges in terms of temporary storage and server load as discussed above) +they can also be configured to use 'direct' upload and download. In this configuration, the actual transfer of file bytes is from/to the user's local machine to/from +the S3 store. In this configuration, files are never stored on the Dataverse server, and Dataverse does not attempt to unzip zip files and they are stored as a single file in the dataset. + +Benefits: S3 offers several advantages over file storage which : + +- Scalability: S3 is designed to handle large amounts of data. It can handle individual files up to several TB in size. +Because S3 supports breaking files into pieces, Dataverse can transfer a file in pieces (several in parallel, potentially thousands of pieces per file) making transfers faster +and more robust (a failure requires only resending the failed piece). It may also be the case that users will have a faster network connection to the S3 store +(e.g. in a commercial cloud or High Performance Computing center) than they do to the Dataverse server, reducing transfer time. +- High Availability: S3 provides redundancy beyond what is available with a single disk (valuable for preservation, potentially reducing the need to perform data integrity checks). + +Challenges: + +- One additional step that is required to enable direct uploads via a Dataverse installation and for direct download to work with previewers and direct upload to work with dvwebloader (:ref:`folder-upload`) is to allow cross site (CORS) requests on your S3 store. +- Cost: S3 offers a pricing model that allows you to pay for the storage and transfer of data based on current usage (versus long term demand) but commercial +providers charge more per TB than the equivalent cost of a local disk (though commercial S3 storage is cheaper than commercial file storage). +There can also be egress and other charges. Overall, S3 storage is generally more expensive than local file storage but cheaper than cloud file storage. +Running a local S3 storage or leveraging an institutional service can further reduce costs. +- Direct upload via S3 is a multi-step process: Dataverse provides URLs for the uploads, the user's browser or other app uses the URLs to transfer files to the S3 store, + possibly in many pieces per file, and finally, Dataverse is told that one or more files are in place and should be added to the dataset. If the last step fails, or if + all parts of a file cannot be transferred, orphaned files or parts of files can be left on S3. These files are not accessible via Dataverse but do use space (for which there is a monetary cost) + until they are deleted. There is currently no automated clean-up mechanism. + +Other Considerations +^^^^^^^^^^^^^^^^^^^^ + +- S3 Storage without direct upload/download provides minimal benefits with Dataverse as files still pass through the server, files are still uploaded as a single HTTP/HTTPS stream, and temporary storage is still used. +- While not having files unzipped can be confusing to users who are used to it from using Dataverse with file storage, there are ways to minimize the impact. +For example, Dataverse can be configured to use a 'Zip File Previewer' that allows users to see the contents of a zip file and even download individual files from within it. +For users who still want their data stored as individual files with their relative folder paths, Dataverse can be configured with the 'DVWebloader' which allows users to select an entire folder tree of files and +upload them, with their relative paths intact, to Dataverse. (DVWebloader can only be used with S3/direct upload, but it is much more efficient with many files than using the +standard upload interface in Dataverse (which also does not retain path information)). +- The following features are disabled when S3 direct upload is enabled. + - Unzipping of zip files. (See :ref:`compressed-files`.) + - Detection of file type based on JHOVE and custom code that reads the first few bytes except for the refinement of Stata file types to include the version. (See :ref:`redetect-file-type`.) + - Extraction of metadata from FITS files. (See :ref:`fits`.) + - Creation of NcML auxiliary files (See :ref:`netcdf-and-hdf5`.) + - Extraction of a geospatial bounding box from NetCDF and HDF5 files (see :ref:`netcdf-and-hdf5`) unless :ref:`dataverse.netcdf.geo-extract-s3-direct-upload` is set to true. +- Using direct upload stops Dataverse from inspecting the file bytes to determine the MIME type (with one exception - Stata files). Dataverse will still look at the file name and extension to determine the MIME type. +- To perform the 'ingest' processing, Dataverse currently has to copy the file to local storage, somewhat negating the benefit of sending data directly to S3. To manage larger files, one can set a per-store +ingest size limit (which can be 0 bytes) to stop ingest or limit it to smaller files. +- Dataverse's mechanism for downloading a whole dataset or multiple selected files involves zipping those files together. Even When using S3 with direct upload/download, +the file bytes are transferred to the Dataverse server as part of the zipping process. There are ways to reduce the performance impact of this: + - There is a 'ZipDownloader' app that can be run separate from Dataverse to handle the zipping process. + - Dataverse has a :ZipDownloadLimit that can be used to limit the amount of data that can be zipped. If a dataset is larger than this limit, Dataverse will only add some of the files to the zip and list others in the included manifest file. + - There are tools such as the Dataverse Dataset Downloader (https://github.com/gdcc/dataverse-recipes/tree/main/shell/download#dataverse-dataset-downloader) that can be used to download all off the files individually. This avoids sending any of the files to/through the Dataverse server when S3 direct download is enabled. +- Dataverse leverages S3 features that are not implemented by all servers and has several configuration options geared towards handling variations between servers. Site admins should be sure to test with their preferred S3 implementation. +- The part-size used when directly transferring files to S3 is configurable (at AWS, from 5 MiB to 5GiB). The default in Dataverse is 1 GiB (1024^3). If the primary use case is with smaller files than that, decreasing the part size may improve upload speeds. + +Remote Stores +~~~~~~~~~~~~~ + +Note: Remote Storage is still experimental: feedback is welcome! See :ref:`support`. + +For very large, and/or very sensitive data, it may not make sense to transfer or copy files to Dataverse at all. +The ``remote`` store type in the Dataverse software supports these use cases. +It allows Dataverse to store a URL reference for the file rather than transferring the file bytes to a store managed directly by Dataverse. +In the most basic configuration a site administrator configures the base URL for the store, e.g. "https://thirdpartystorage.edu/long-term-storage/" +and users can then create files referencing any URL starting with that base, e.g. "https://thirdpartystorage.edu/long-term-storage/my_project_dir/my_file.txt". + +If the remote site is a public web server, the remote store in Dataverse should be configured to be 'public' which will disable the ability to restrict +or embargo files (as they are public on the remote site and Dataverse cannot block access.) Conversely, Dataverse can be configured to sign requests to the +remote server which the remote server can then, if it is capable of validating them, use to reject requests not approved by Dataverse. In this configuration, +users can restrict and embargo files and Dataverse and the remote server will cooperate to manage access control. Another alternative, with a more advanced +remote store, would be, instead of using URLs that directly enable download of the file, to use URLs that point to a landing page at the remote server that +may require the user to login, or go through some other authentication/validation process before being able to access the file. + +Dataverse considers remote storage to be read-only, or, in cases where the remote service does not provide a way for Dataverse to download the file bytes +(due to access control or because the URL refers to a landing page), inacessible. Depending on whether Dataverse can access the bytes of the file, +functionality such as ingest and integrity checking may or may not be possible. If the file bytes are not accessible the remote store in Dataverse should be +configured to disable operations that attempt to access the file (see files-not-accessible-by-dataverse). If the file bytes are accessible, Dataverse can still +support features such as ingest and thumbnail creation in Dataverse-local storage. In either case, local storage of other files and auxiliary files in the same +dataset is possible. Support for such files is handled by configuring a 'base' store with the remote store that is used for these purposes. (This means that while +the specified files remain on the remote store, other files in the dataset, and potentially the ingested TSV format of a remote file would be managed by Dataverse +in some other store. If ingest is not desired, the ingest size limit for the store can be set to 0 bytes). + +Benefits: + +- This is a relatively simple way to off-load the management of large and/or sensitive data files to other organizations while still providing Dataverse's overall capabilities for dataset curation and publication to users. +- If the store has been configured with a remote-store-name or remote-store-url, the dataset file table will include this information for remote files. These provide a visual indicator that the files are not managed directly by Dataverse and are stored/managed by a remote trusted store. + +Challenges: + +- Currently, remote files can only be added via the API. (This may be addressed in future versions). +- Remote files can only be added after the dataset is created in the UI (and therefore has an id and PID for use with the API). However, the UI will still allow upload of files to the base store (at dataset creation and when editing), which could be confusing. +- As Dataverse is relying on the remote service to main the integrity and availability of the files, it is likely that the Dataverse site admin will want to have a formal agreement with the remote service +operator about their policies. +- Site admins need to consider carefully how to configure file size limits, ingest size limits, etc. on the remote store and it's base store, and whether the remote store is public-only, and whether files there can be read by Dataverse to assure the +requirements of a specific use case(s) are addressed. +- The current remote store implementation will not prevent you from providing a relative URL that results in a 404 when resolved. (I.e. if you make a typo). You should check to make sure the file exists at the location you specify - by trying to download in Dataverse, by checking to see that Dataverse was able to get the file size (which it does by doing a HEAD call to that location), or just manually trying the URL in your browser. +- For large files, direct-download should always be used with a remote store. (Otherwise the Dataverse will be involved in the download.) +- When multiple files are selected for download, Dataverse will try to include remote files in the zip file being created (up to the max zip size limit) which is inefficient, and will not be able to include remote files that are inaccessible (possibly confusing). + +Globus Stores: Globus Transfer between Large File/Tape Archives +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Note: Globus Transfer is still experimental: feedback is welcome! See :ref:`support`. + +`Globus `_ provides file transfer service that is widely used for the world's largest datasets (in terms of both file size and number of files). It provides: + +- robust file transfer capable of handling delays (e.g. due to the time it takes to mount tapes) and restarting after network or endpoint failures +- rapid parallel file transfers, potentially between clusters of computers on both ends +- third-party transfer, which enables a user working with their desktop browser to initiate transfer of files from one remote endpoint (i.e. on a local high-performance computing cluster) to/from another (e.g. one associated with a Dataverse store) + +Dataverse can be configured to support Globus transfers in multiple ways: + +- A Dataverse-managed Globus File Endpoint: Dataverse controls user access to the endpoint, access is only via Globus +- A Dataverse-managed Globus S3 Endpoint: Dataverse controls user access to the endpoint, access is available via S3 and via Globus +- A Globus Endpoint treated as Remote Storage: Dataverse references files on a Globus endpoint managed by a third party + +Benefits: + +- Globus scales to higher data volumes than any other option. Users working with large data are often familiar with Globus and are interested in transferring data to/from computational clusters rather than their local machine +- Globus transfers can be initiated by choosing the Globus option in the dataset upload panel. Analogously, "Globus Transfer" is one of the download options in the "Access Dataset" menu. +- For the non-S3 options, Dataverse supports having a base store (e.g. a local file system or an S3-based store), which can be used internally by Dataverse (e.g. for thumbnails, etc.) and can allow users to upload smaller files (e.g. Readmes, documentation) that might not be suited to a given Globus endpoint (e.g. a tape store) + +Challenges: + +- Globus is complex to manage and Dataverse installations will need to develop Globus expertise or partner with another organization (i.e. a institutional high-performance computing center) to manage Globus endpoints. +- For users not familiar with Globus, managing transfers can be confusing. For the non-S3 options, users cannot just download files - they must have access to a destination Globus endpoint and have a Globus account. Globus does provide free accounts and a free 'Globus Personal Connect' service which installed on any machine to allow transfers to/from it. +- Globus transfers are not enabled at dataset-creation time. Once the draft version is created, users can initiate Globus transfers to upload files from remote endpoints. +- For Dataverse-managed endpoints, a community-developed `dataverse-globus `_ app must be installed and configured in the Dataverse instance. +This app manages granting and revoking access for users to upload/download files from Dataverse and handles the translation between Dataverse's internal file naming/organization to that see by the user. +- Users familiar with Globus sometimes expect to be able to find the Dataverse endpoint in Globus' online service and download files from there. Due to the fact that Dataverse is managing permissions and handling file naming, this doesn't work. +- Due to differences between Dataverse's and Globus's access control models, Dataverse cannot enforce per-file-access restrictions - restriction can only be done today at the level of providing access to all files in a dataset. +Globus stores can be defined as public to disable Dataverse's ability to restrict and embargo files in that store. If the store is configured to support restriction and embargo, +Dataverse and it's Dataverse-Globus app will limit users to downloading only the files they have been granted access to, but a technically knowledgeable user could access other files in the same dataset if they are give access to one. +(Data depositors would need to be aware of this limitation and could be guided to restrict all files/only grant access to all dataset files in Globus as a work-around). +- Dataverse-managed endpoints must be Globus 'guest collections' hosted on either a file-system-based endpoint or an S3-based endpoint (the latter requires use of the Globus +S3 connector which requires a paid Globus subscription at the host institution). In either case, Dataverse is configured with the Globus credentials of a user account that can manage the endpoint. +Users will need their own Globus account, which can be obtained via their institution or directly from Globus (at no cost). +- With the file-system endpoint, Dataverse does not currently have access to the file contents. Thus, functionality related to ingest, previews, fixity hash validation, etc. are not available. +(Using the S3-based endpoint, Dataverse has access via S3 and all functionality normally associated with direct uploads to S3 is available. In this case admins should be sure to set the maximum size for ingest and avoid requiring hash validation at publication, etc.) +- For the reference use case, Dataverse must be configured with a list of allowed endpoint/base paths from which files may be referenced. In this case, since Dataverse is not accessing the remote endpoint itself, it does not need Globus credentials. +Users will also need a Globus account in this case, and the remote endpoint must be configured to allow them access (i.e. be publicly readable, or potentially involving some out-of-band mechanism to request access (that could be described, for example, in the dataset's Terms of Use and Access). +- As with remote stores, files can only be added in the Globus reference case via the Dataverse API. +- While Globus itself can handle many (millions of) files of any size, Dataverse cannot handle more than thousands of files per dataset (at best) and some Globus endpoints may have limits on file sizes - both maximums and minimums (e.g. for tape storage where small files are inefficient). +Users will need to be made aware of these limitations and the possibilities for managing them (e.g. by aggregating multiple files in a single larger file, or storing smaller files in the base-store via the normal Dataverse upload UI). +- There is currently (a bug)[https://github.com/gdcc/dataverse-globus/issues/2] that won't allow users to transfer files from/to endpoints where they do not have permission to list the overall file tree (i.e. an institution manages /institution_name but the user only has access to /institution_name/my_dir.) +Until that is fixed, a work-around is to first transfer data to an endpoint without this restriction. +- An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. While it is now the recommended option, it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag (see :ref:`feature-flags`) and the JVM option :ref:`dataverse.files.globus-monitoring-server`. + +More details of the setup required to enable Globus is described in the `Community Dataverse-Globus Setup and Configuration document `_ and the references therein. + +An overview of the control and data transfer interactions between components was presented at the 2022 Dataverse Community Meeting and can be viewed in the `Integrations and Tools Session Video `_ around the 1 hr 28 min mark. + +See also :ref:`Globus settings <:GlobusSettings>`. + + +Storage Strategy Recommendations +------------------------------- + +Based on both file size and volume considerations, here are some general recommendations: + +1. **For research projects with moderate data (< 2GB files, < 100s of files/dataset):** + * Default File Store is sufficient + * Consider setting file count and size limits (see below) + +2. **For projects with larger files (GBs to TBs) and/or more files per dataset (100s to 1000s):** + * Configure an S3 store witj direct upload/download + * Set appropriate ingest size limits + +3. **For projects with very large files or sensitive data that should remain in place:** + * Use the Remote Store + +4. **For high-performance computing environments or very large datasets (TBs+):** + * Use a Globus Store + * Work with users to size files appropriate to the underlying storage + * Consider Globus over S3 when normal upload/download options (via the UI/API) are desired along with Globus transfer + +5. **For Petascale datasets, or extreme numbers of files:** + * Consider a Remote Store and referencing a single URL/Globus endpoint for the entire dataset + + +Managing more files per dataset and more datasets +------------------------------------------------- + +Dataverse can be configured to handle datasets with hundreds or thousands of files and hundreds of thousands of datasets. However, reaching these levels can require significant effort to appropriately configure the server. + +Technically, there are two factors that can limit scaling the number of files per dataset: how much the Dataverse server is involved in data transfer, and constraints based on Dataverse's code and database configuration. +The former is dramatically affected by the choice for file storage and options such as the S3 direct upload/download settings and ingest size limits. There are fewer ways to affect the latter beyond increasing the amount of memory and CPU resources available +or rewriting the relevant parts of Dataverse. (There are continuing efforts to improve Dataverse's performance and scaling, so it is also advisable to use the latest version if you are pushing the boundaries on scaling. Progress is being made.) + +Scaling to larger numbers of datasets (and to some extent scaling files per dataset) also depends on Dataverse's Solr search engine. There have been very significant improvements in indexing and search performance in recent releases, including some that are not turned on by default. + +Avoiding Many Files +~~~~~~~~~~~~~~~~~~~ + +Before describing things that can be done to improve scaling, it is important to note that there are configuration options and best practices to suggest to users to help avoid larger datasets and and help them avoid hitting performance issues by going beyond the file counts you know work in your instance. + +There are a number of settings to limit how many files can be uploaded: + +- ZipUploadFilesLimit - the maximum number of files allowed in an uploaded zip file - only relevant for file stores and S3 when direct upload is not used. +- MultipleUploadFilesLimit - the number of files the GUI user is allowed to upload in one batch, via drag-and-drop, or through the file select dialog +- MaxFileUploadSizeInBytes - limit the size of files that can be uploaded + +- UseStorageQuotas - once enabled, super users can set per-collection quotas (in bytes) to limit the aggregate size of all files in the collection +- dataverse.files.default-dataset-file-count-limit - directly limits the number of files per dataset, can be changed per dataset via API (by super users) + +Ways to Scale +~~~~~~~~~~~~~ + +There are a broad range of options (that are not turned on by default) for improving how well Solr indexing and searching scales. Some of these are useful for all installations while others are related to specific use cases, or are mostly for emergency use (e.g. disabling facets): + +- dataverse.feature.add-publicobject-solr-field=true - specifically marks unrestricted content as public in solr +- dataverse.feature.avoid-expensive-solr-join=true - this tells Dataverse to use the feature above to speed searches +- dataverse.feature.reduce-solr-deletes=true - when solr entries are being updated, this avoids an unnecessary step (deletion of existing entries) for entries that are being replaced +- dataverse.feature.disable-dataset-thumbnail-autoselect=true - by default, Dataverse scans through all files in a dataset to find one that can be used as a thumbnail, which is expensive for many files. This disables that behavior to improve performance +- dataverse.feature.only-update-datacite-when-needed=true - reduces the load on DataCite and reduces Dataverse failures related to that load, which is important when using file PIDs on Datasets with many files +- v6.9: dataverse.solr.min-files-to-use-proxy= - improve performance/lower memory requirements when indexing datasets with many files, suggested value is in the range 200 to 500 +- dataverse.solr.concurrency.max-async-indexes= - limits the number of index operations running in parallel. The default is 4, larger values may improve performance (if the Solr intance is appropriately sized) +- v6.9?: dataverse.exports.schema-dot-org.max-files-for-download-entries - reduces the size of the schema.org metadata export wehn there are many files per dataset. By default, this is included in the dataset page header, so the smaller version improves page loading and can avoid issues with Google indexing (datasets with thousands of files+) +- SolrFullTextIndexing - false improves performance at the expense of not indexing file contents +- SolrMaxFileSizeForFullTextIndexing - size in bytes (default unset/no limit) above which file contents should not be indexed +- ZipDownloadLimit - the maximum size in bytes for zipped downloads of files from a dataset. If the size of requested files is larger, some files will be omitted and listedin the zip manifest file as not included. +- DatasetChecksumValidationSizeLimit - by default, Dataverse checks fixity (assuring the file contents match the recorded checksum) as part of publication. This setting specifies a maximum aggregate dataset size, above which validation will not be done. +- DataFileChecksumValidationSizeLimit - by default, Dataverse checks fixity (assuring the file contents match the recorded checksum) as part of publication. This setting specifies a maximum file size, above which validation will not be done. +- FilePIDsEnabled - false is recommended when datasets have many files. Related settings allow file PIDS to be enabled/disabled per collection and per file +- CustomZipDownloadServiceUrl - allows use of a separate process/machine to handle zipping up multi-file downloads. Requires installation of the separate Zip Download app. +- WebloaderUrl - enables use of an installed DVWebloader (by specifying it's web location) which is more efficient for uploading many files +- CategoryOrder - Pre-sorts the file display by category, e.g. showing all "Documentation" files before "Data" files. Any user selected sorting by name, age, or size is done within these sections +- OrderByFolder - pre-sorts files by their directory Label (folder), showing files with no path before others. Any user selected sorting by name, age, or size is done within these sections +- DisableSolrFacets - disables facets, which are costly to generate, in search results (including the main collection page) +- DisableSolrFacetsForGuestUsers - only disable facets for guests +- DisableSolrFacetsWithoutJsession - disables facets for users who have disabled cookies (e.g. for bots) +- DisableUncheckedTypesFacet -only disables the facet showing the number of collections, datasets, files matching the query (this facet is potentially less useful than others) +- StoreIngestedTabularFilesWithVarHeaders - by default, Dataverse stores ingested files without headers and dynamically adds them back at download time. Once this setting is enabled, Dataverse will leave the headers in place (for newly ingested files), reducing the cost of downloads + + +Scaling Infrastructure +---------------------- + +There is no well-defined cut-off in terms of files per dataset or number of datasets where the Dataverse software will fail. In general the speed of viewing and editing a large dataset will decrease as the volume of datasets and files increases. +For a given installation, at some point, Dataverse will need more memory than is available, or will max out the CPU or other resources and performance may decline dramatically. + +In such cases: + +- Consider increasing the memory available to Dataverse (the Java heap size for the Payara instance) +- Consider a larger machine (more CPU resources) +- Verify that performance isn't being limited by Solr or Postgres +- Investigate performance tuning options for Payara, Solr, and Postgres +- Coordinate with others in the community - there is a lot of aggregate knowledge +- Consider contributing to software design changes - Dataverse scaling has improved dramatically over the past several years, but more can be done +- Watch for the new single page application (SPA) front-end for Dataverse. It includes features such as infinite scrolling through files with much faster initial page load times diff --git a/doc/sphinx-guides/source/admin/index.rst b/doc/sphinx-guides/source/admin/index.rst index a8a543571a7..4d2d5c22fc2 100755 --- a/doc/sphinx-guides/source/admin/index.rst +++ b/doc/sphinx-guides/source/admin/index.rst @@ -35,3 +35,4 @@ This guide documents the functionality only available to superusers (such as "da maintenance backups troubleshooting + big-data-administration From 67977c2cd7454db0384e61bab74bf6e4739cdca4 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 26 Sep 2025 15:04:25 -0400 Subject: [PATCH 306/634] restore big data dev guide --- .../source/developers/big-data-support.rst | 444 +----------------- 1 file changed, 17 insertions(+), 427 deletions(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 1c388a7c565..75a50e2513d 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -1,354 +1,28 @@ -Scaling Dataverse with Data Size -================================ - -This guide is intended to help administrators configure Dataverse appropriately to handle the amount of data their installation needs to manage. - -Scaling is a complex subject: there are many options available in Dataverse that can improve performance with larger scale data, some of which -work differently than Dataverse's default configuration, potentially requiring user education, and some which can require additional expertise to manage. - -In general, there are three dimensions in which Dataverse can scale: - -1. **Storage size of individual files and aggregate storage size** -2. **Number of Files per Dataset** -3. **Number of Datasets** +Big Data Support +================ +Big data support includes some experimental options. Eventually more of this content will move to the Installation Guide. .. contents:: |toctitle| :local: -Choosing the Right Storage --------------------------- - -The main issues for handling larger files and larger aggregate data size relate to the performance of the storage used and how involved the Dataverse server is in the data transfer. -With appropriate configuration, Dataverse can support file sizes and aggregate dataset sizes into the terabyte scale and beyond. - -File Storage -~~~~~~~~~~~~ - -The default storage option in Dataverse is based on a local file system. When files are transferred to Dataverse, they are first stored in a -temporary location on the Dataverse server. Any zip files uploaded are unzipped to create multiple individual file entries. Once an upload is completed, -Dataverse copies the files to permanent storage. Dataverse also takes advantage of the file being local to inspect it's bytes to determine it -s MIME type, and, for tabular data, to 'ingest' it - extracting metadata about the variables used in the file and creating a tab-separated value (TSV) version of the file. - -Benefits: - -- This option requires no external services and can potentially handle files into the gigabyte (GB) size range. For smaller institutions, -and in disciplines where datasets do not have more than a few hundred files and files are not too large, this can be the simplest option. -- Unzipping of zip archives can be simpler for users than having to upload many individual files and was, at one time, the only way to -preserve file path names when uploading. - -Challenges: In general file storage is not a good option for larger data sizes - both in terms of file size and number of files. Contributing factors include: - -- Because temporary storage is used, transfers will temporarily use several times as much space as the final transfer. Unzipping also increases the final storage size of a dataset. -- Because all uploads use the same temporary storage, temporary storage must be large enough to handle multiple users uploading data. -- Each file is uploaded as a single HTTP request, which can cause long transfer times which, in turn, can trigger timeout errors in Dataverse or any proxy or load balancer in front of Dataverse. -- Uploading many files at once can trigger any rate limiter in front of the Dataverse server (i.e. used to throttle use by AI) resulting in failures -- Because transfers (both uploads and downloads) are handled by the Dataverse server, they add to server processing load which can affect overall performance. -- Cost: local file storage must be provisioned in advance based on anticipated demand. It can involve up-front costs (for a local disk), or, when procured from a cloud provider, is likely to be more expensive than object storage (see below) - - -Object Storage via S3 -~~~~~~~~~~~~~~~~~~~~~ - -A more scalable option for storage is to use an object store with transfers managed using the Simple Storage Service (S3) protocol. S3-compatible storage can -easily be bought (rented) from major cloud providers, but may also be available from institutional clouds. It is also possible to run open-source software to provide S3 storage -over a local file system (making it possible to enjoy the advantages discussed below while still leveraging local file storage). - -While Dataverse can be configured to handle uploads and downloads as with file storage (with zip files being unzipped, but having many of the same challenges in terms of temporary storage and server load as discussed above) -it can also be configured to use 'direct' upload and download. In this configuration, the actual transfer of file bytes is from/to the user's local machine to/from -the S3 store. In this configuration, Dataverse does not attempt to unzip zip files and they are stored as a single file in the dataset. - -Benefits: S3 offers several advantages over file storage which : - -- Scalability: S3 is designed to handle large amounts of data. It can handle individual files up to several TB in size. -Because S3 supports breaking files into pieces, Dataverse can transfer a file in pieces (several in parallel, potentially thousands of pieces per file) making transfers faster -and more robust (a failure requires only resending the failed piece). It may also be the case that users will have a faster network connection to the S3 store -(e.g. in a commercial cloud or High Performance Computing center) than they do to the Dataverse server, reducing transfer time. -- High Availability: S3 provides redundancy beyond what is available with a single disk (valuable for preservation, potentially reducing the need to perform data integrity checks). - -Challenges: - -- Cost: S3 offers a pricing model that allows you to pay for the storage and transfer of data based on current usage (versus long term demand) but commercial -providers charge more per TB than the equivalent cost of a local disk (though commercial S3 storage is cheaper than commercial file storage). -There can also be egress and other charges. Overall, S3 storage is generally more expensive than local file storage but cheaper than cloud file storage. -Running a local S3 storage or leveraging an institutional service can further reduce costs. -- S3 Storage without direct upload/download provides minimal benefits with Dataverse as files still pass through the server, files are still uploaded as a single HTTP/HTTPS stream, and temporary storage is still used. - -Other Considerations -^^^^^^^^^^^^^^^^^^^^ - -- While not having files unzipped can be confusing to users who are used to it from using Dataverse with file storage, there are ways to minimize the impact. -For example, Dataverse can be configured to use a 'Zip File Previewer' that allows users to see the contents of a zip file and even download individual files from within it. -For users who still want their data stored as individual files, Dataverse can be configured with the 'DVWebloader' which allows users to select an entire folder tree of files and -upload them, with their relative paths intact, to dataverse. (DVWebloader can only be used with S3/direct upload, but it is much more efficient in this case than using the -standard upload interface in Dataverse). -- Using direct upload stops Dataverse from inspecting the file bytes to determine the MIME type (with one exception - Stata files). Dataverse will still look at the file name and extension to determine the MIME type. -- To perform the 'ingest' processing, Dataverse currently has to copy the file to local storage, somewhat negating the benefit of sending data directly to S3. To manage larger files, one can set a per-store -ingest size limit (which can be 0 bytes) to stop ingest or limit it to smaller files. -- Dataverse's mechanism for downloading a whole dataset or multiple selected files involves zipping those files together. Even When using S3 with direct upload/download, -the file bytes are transferred to the Dataverse server as part of the zipping process. There are ways to reduce the performance impact of this: - - There is a 'ZipDownloader' app that can be run separate from Dataverse to handle the zipping process. - - Dataverse has a :ZipDownloadLimit that can be used to limit the amount of data that can be zipped. If a dataset is larger than this limit, Dataverse will only add some of the files to the zip and list others in the included manifest file. - - There are tools such as the Dataverse Dataset Downloader (https://github.com/gdcc/dataverse-recipes/tree/main/shell/download#dataverse-dataset-downloader) that can be used to download all off the files individually. This avoids sending any of the files to/through the Dataverse server when S3 direct download is enabled. -- Dataverse leverages S3 features that are not implemented by all servers and has several configuration options geared towards handling variations between servers. Site admins should be sure to test with their preferred S3 implementation. -- The part-size used when directly transferring files to S3 is configurable (at AWS, from 5 MiB to 5GiB). The default in Dataverse is 1 GiB (1024^3). If the primary use case is with smaller files than that, decreasing the part size may improve upload speeds. - -Remote Storage -~~~~~~~~~~~~~~ - -Note: Remote Storage is still experimental: feedback is welcome! See :ref:`support`. - -For very large, and/or very sensitive data, it may not make sense to transfer or copy files to Dataverse at all. -The ``remote`` store type in the Dataverse software supports these use cases. -It allows Dataverse to store a URL reference for the file rather than transferring the file bytes to a store managed directly by Dataverse. -In the most basic configuration a site administrator configures the base URL for the store, e.g. "https://thirdpartystorage.edu/long-term-storage/" -and users can then create files referencing any URL starting with that base, e.g. "https://thirdpartystorage.edu/long-term-storage/my_project_dir/my_file.txt". -If the remote site is a public web server, the remote store in Dataverse should be configured to be 'public' which will disable the ability to restrict -or embargo files (as they are public on the remote site and Dataverse cannot block access.) Conversely, Dataverse can be configured to sign requests to the -remote server which the remote server can then, if it is capable of validating them, use to reject requests not approved by Dataverse. In this configuration, -users can restrict and embargo files and Dataverse and the remote server will cooperate to manage access control. Another alternative, with a more advanced -remote store, would be, instead of using URLs that directly enable download of the file, to use URLs that point to a landing page at the remote server that -may require the user to login, or go through some other authentication/validation process before being able to access the file. - -Dataverse considers remote storage to be read-only, or, in cases where the remote service does not provide a way for Dataverse to download the file bytes -(due to access control or because the URL refers to a landing page), inacessible. Depending on whether Dataverse can access the bytes of the file, -functionality such as ingest and integrity checking may or may not be possible. If the file bytes are not accessible the remote store in Dataverse should be -configured to disable operations that attempt to access the file (see files-not-accessible-by-dataverse). If the file bytes are accessible, Dataverse can still -support features such as ingest and thumbnail creation, as well as local storage of other files and auxilliary files. These are handled by configuring a 'base' store -with the remote store that is used for these purposes. (This means that while the specified files remain on the remote store, other files in the dataset, and -potentially the ingested TSV format of a remote file would be managed by Dataverse in some other store. If ingest is not desired, the ingest size limit for the -store can be set to 0 bytes). - -Benefits: - -- This is a relatively simple way to off-load the management of large and/or sensitive data files to other organizations while still providing Dataverse's overall capabilities for dataset curation and publication to users. -- If the store has been configured with a remote-store-name or remote-store-url, the dataset file table will include this information for remote files. These provide a visual indicator that the files are not managed directly by Dataverse and are stored/managed by a remote trusted store. - -Challenges: - -- Currently, remote files can only be added via the API. (This may be addressed in future versions). -- Since remote files can only be added as the dataset is created in the UI. However, the UI will still allow upload of files to the base store (at dataset creation and when editing) -- As Dataverse is relying on the remote service to main the integrity and availability of the files, it is likely that the Dataverse site admin will want to have a formal agreement with the remote service -operator about their policies. -- Site admins need to consider carefully how to configure file size limits, ingest size limits, etc. on the remote store and it's base store, and whether the remote store is public-only, and whether it files there can be read by Dataverse to assure the -requirements of a specific use case(s) are addressed. -- The current remote store implementation will not prevent you from providing a relative URL that results in a 404 when resolved. (I.e. if you make a typo). You should check to make sure the file exists at the location you specify - by trying to download in Dataverse, by checking to see that Dataverse was able to get the file size (which it does by doing a HEAD call to that location), or just manually trying the URL in your browser. -- For large files, direct-download should always be used with a remote store. (Otherwise the Dataverse will be involved in the download.) -- When multiple files are selected for download, Dataverse will try to include remote files in the zip file being created (up to the max zip size limit) which is inefficent, and will not be able to include remote files that are inaccessible (possibly confusing). - -Globus Transfer -~~~~~~~~~~~~~~~ - -Note: Globus Transfer is still experimental: feedback is welcome! See :ref:`support`. - -`Globus `_ provides file transfer service that is widely used for the world's largest datasets (in terms of both file size and number of files). It provides: - -- robust file transfer capable of handling delays (e.g. due to the time it takes to mount tapes) and restarting after network or endpoint failures -- rapid parallel file transfers, potentially between clusters of computers on both ends -- third-party transfer, which enables a user working with their desktop browser to initiate transfer of files from one remote endpoint (i.e. on a local high-performance computing cluster) to/from another (e.g. one associated with a Dataverse store) - -Dataverse can be configured to support Globus transfers in multiple ways: - -- A Dataverse-managed Globus File Endpoint: Dataverse controls user access to the endpoint, access is only via Globus -- A Dataverse-managed Globus S3 Endpoint: Dataverse controls user access to the endpoint, access is available via S3 and via Globus -- A Globus Endpoint treated as Remote Storage: Dataverse references files on a Globus endpoint managed by a third party - -Benefits: - -- Globus scales to higher data volumes than any other option. Users working with large data are often familiar with Globus and are interested in transferring data to/from computational clusters rather than their local machine -- Globus transfers can be initiated by choosing the Globus option in the dataset upload panel. Analogously, "Globus Transfer" is one of the download options in the "Access Dataset" menu. -- For the non-S3 options, Dataverse supports having a base store (e.g. a local file system or an S3-based store), which can be used internally by Dataverse (e.g. for thumbnails, etc.) and can allow users to upload smaller files (e.g. Readmes, documentation) that might not be suited to a given Globus endpoint (e.g. a tape store) - -Challenges: - -- Globus is complex to manage and Dataverse installations will need to develop Globus expertise or partner with another organization (i.e. a institutional high-performance computing center) to manage Globus endpoints. -- For users not familiar with Globus, managing transfers can be confusing. For the non-S3 options, users cannot just download files - they must have access to a destination Globus endpoint. Globus does provide a free 'Globus Personal Connect' service which installed on any machine to allow transfers to/from it. -- Globus transfers are not enabled at dataset-creation time. Once the draft version is created, users can initiate Globus transfers to upload files from remote endpoints. -- For Dataverse-managed endpoints, a community-developed `dataverse-globus `_ app must be installed and configured in the Dataverse instance. -This app manages granting and revoking access for users to upload/download files from Dataverse and handles the translation between Dataverse's internal file naming/organization to that see by the user. -- Users familiar with Globus sometimes expect to be able to find the Dataverse endpoint in Globus' online service and download files from there. Due to the fact that Dataverse is managing permissions and handling file naming, this doesn't work. -- Due to differences between Dataverse's and Globus's access control models, Dataverse cannot enforce per-file-access restrictions - restriction can only be done today at the level of providing access to all files in a dataset. -Globus stores can be defined as public to disable Dataverse's ability to restrict and embargo files in that store. If the store is configured to support restriction and embargo, -Dataverse and it's Dataverse-Globus app will limit users to downloading only the files they have been granted access to, but a technically knowledgeable user could access other files in the same dataset if they are give access to one. -(Data depositors would need to be aware of this limitation and could be guided to restrict all files/only grant access to all dataset files as a work-around). -- Dataverse-managed endpoints must be Globus 'guest collections' hosted on either a file-system-based endpoint or an S3-based endpoint (the latter requires use of the Globus -S3 connector which requires a paid Globus subscription at the host institution). In either case, Dataverse is configured with the Globus credentials of a user account that can manage the endpoint. -Users will need their own Globus account, which can be obtained via their institution or directly from Globus (at no cost). -- With the file-system endpoint, Dataverse does not currently have access to the file contents. Thus, functionality related to ingest, previews, fixity hash validation, etc. are not available. -(Using the S3-based endpoint, Dataverse has access via S3 and all functionality normally associated with direct uploads to S3 is available. In this case admins should be sure to set the maximum size for ingest and avoid requiring hash validation at publication, etc.) -- For the reference use case, Dataverse must be configured with a list of allowed endpoint/base paths from which files may be referenced. In this case, since Dataverse is not accessing the remote endpoint itself, it does not need Globus credentials. -Users will also need a Globus account in this case, and the remote endpoint must be configured to allow them access (i.e. be publicly readable, or potentially involving some out-of-band mechanism to request access (that could be described, for example, in the dataset's Terms of Use and Access). -- While Globus itself can handle many (millions of) files of any size, Dataverse cannot handle more than thousands of files per dataset (at best) and some Globus endpoints may have limits on file sizes - both maximums and minimums (e.g. for tape storage where small files are inefficient). -Users will need to be made aware of these limitations and the possibilities for managing them (e.g. by aggregating multiple files in a single larger file, or storing smaller files in the base-store via the normal Dataverse upload UI). -- There is currently (a bug)[https://github.com/gdcc/dataverse-globus/issues/2] that won't allow users to transfer files from/to endpoints where they do not have permission to list the overall file tree (i.e. an institution manages /institution_name but the user only has access to /institution_name/my_dir.) -Until that is fixed, a work-around is to first transfer data to an endpoint without this restriction. -- An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. While it is now the recommended option, it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag (see :ref:`feature-flags`) and the JVM option :ref:`dataverse.files.globus-monitoring-server`. +Various components will need to be installed and/or configured for big data support via the methods described below. -More details of the setup required to enable Globus is described in the `Community Dataverse-Globus Setup and Configuration document `_ and the references therein. +S3 Direct Upload and Download +----------------------------- -An overview of the control and data transfer interactions between components was presented at the 2022 Dataverse Community Meeting and can be viewed in the `Integrations and Tools Session Video `_ around the 1 hr 28 min mark. +A lightweight option for supporting file sizes beyond a few gigabytes - a size that can cause performance issues when uploaded through a Dataverse installation itself - is to configure an S3 store to provide direct upload and download via 'pre-signed URLs'. When these options are configured, file uploads and downloads are made directly to and from a configured S3 store using secure (https) connections that enforce a Dataverse installation's access controls. (The upload and download URLs are signed with a unique key that only allows access for a short time period and a Dataverse installation will only generate such a URL if the user has permission to upload/download the specific file in question.) -See also :ref:`Globus settings <:GlobusSettings>`. +This option can handle files >300GB and could be appropriate for files up to a TB or larger. Other options can scale farther, but this option has the advantages that it is simple to configure and does not require any user training - uploads and downloads are done via the same interface as normal uploads to a Dataverse installation. -Managing more files per dataset and more datasets -------------------------------------------------- - -Dataverse can be configured to handle datasets with hundreds or thousands of files and hundreds of thousands of datasets. However, reaching these levels can require significant effort to appropriately configure the server. - -Technically, there are two factors that can limit scaling the number of files per dataset: how much the Dataverse server is involved in data transfer, and constraints based on Dataverse's code and database configuration. -The former is dramatically affected by the choice for file storage and options such as the S3 direct upload/download settings and ingest size limits. The are fewer ways to affect the latter beyond increasing the amount of memory and CPU resources available -or rewriting the relevant parts of Dataverse. (There are continuing efforts to improve Dataverse's performance and scaling, so it is also advisable to use the latest version if you are pushing the boundaries on scaling. Progress is being made.) - -Scaling to larger numbers of datasets (and to some extent scaling files per dataset) also depends on Dataverse's Solr search engine. There have been very significant improvements in indexing and search performance in recent releases, including some that are not turned on by default. - -Avoiding Many Files -~~~~~~~~~~~~~~~~~~~ - -Before describing things that can be done to improve scaling, it is important to note that there are configuration options and best practices to suggest to users to help avoid larger datasets and users hitting performance issues by going beyond the file counts you know work in your instance. - - /* zip upload number of files limit */ - ZipUploadFilesLimit, - /* the number of files the GUI user is allowed to upload in one batch, - via drag-and-drop, or through the file select dialog */ - MultipleUploadFilesLimit, - UseStorageQuotas, - /** - * Placeholder storage quota (defines the same quota setting for every user; used to test the concept of a quota. - */ - StorageQuotaSizeInBytes, - -Ways to Scale -~~~~~~~~~~~~~ - - - /** - * For published (public) objects, don't use a join when searching Solr. - * Experimental! Requires a reindex with the following feature flag enabled, - * in order to add the boolean publicObject_b:true field to all the public - * Solr documents. - * - * @apiNote Raise flag by setting - * "dataverse.feature.avoid-expensive-solr-join" - * @since Dataverse 6.3 - */ -AVOID_EXPENSIVE_SOLR_JOIN("avoid-expensive-solr-join"), - /** - * With this flag enabled, the boolean field publicObject_b:true will be - * added to all the indexed Solr documents for publicly-available collections, - * datasets and files. This flag makes it possible to rely on it in searches, - * instead of the very expensive join (the feature flag above). - * - * @apiNote Raise flag by setting - * "dataverse.feature.add-publicobject-solr-field" - * @since Dataverse 6.3 - */ - ADD_PUBLICOBJECT_SOLR_FIELD("add-publicobject-solr-field"), - /** - * Dataverse normally deletes all solr documents related to a dataset's files - * when the dataset is reindexed. With this flag enabled, additional logic is - * added to the reindex process to delete only the solr documents that are no - * longer needed. (Required docs will be updated rather than deleted and - * replaced.) Enabling this feature flag should make the reindex process - * faster without impacting the search results. - * - * @apiNote Raise flag by setting - * "dataverse.feature.reduce-solr-deletes" - * @since Dataverse 6.3 - */ - REDUCE_SOLR_DELETES("reduce-solr-deletes"), - /** - * This flag disables the feature that automatically selects one of the - * DataFile thumbnails in the dataset/version as the dedicated thumbnail - * for the dataset. - * - * @apiNote Raise flag by setting - * "dataverse.feature.disable-dataset-thumbnail-autoselect" - * @since Dataverse 6.4 - */ - DISABLE_DATASET_THUMBNAIL_AUTOSELECT("disable-dataset-thumbnail-autoselect"), - /** - * Only update a DataCite DOI when needed (for efficiency, lighter load on DataCite). - */ - ONLY_UPDATE_DATACITE_WHEN_NEEDED("only-update-datacite-when-needed"), - - -JvmSettings: - MIN_FILES_TO_USE_PROXY(SCOPE_SOLR, "min-files-to-use-proxy"), - - // INDEX CONCURENCY - SCOPE_SOLR_CONCURENCY(SCOPE_SOLR, "concurrency"), - MAX_ASYNC_INDEXES(SCOPE_SOLR_CONCURENCY, "max-async-indexes"), - - //EXPORTS - SCOPE_EXPORTS(PREFIX, "exports"), - SCOPE_EXPORTS_SCHEMA_DOT_ORG(SCOPE_EXPORTS, "schema-dot-org"), - EXPORTS_SCHEMA_DOT_ORG_MAX_FILES_FOR_DOWNLOAD_ENTRIES(SCOPE_EXPORTS_SCHEMA_DOT_ORG, "max-files-for-download-entries"), - -Settings: - - SolrFullTextIndexing, //true or false (default) - SolrMaxFileSizeForFullTextIndexing, //long - size in bytes (default unset/no limit) - /** Default Key for limiting the number of bytes uploaded via the Data Deposit API, UI (web site and . */ - MaxFileUploadSizeInBytes, - ZipDownloadLimit, -/* size limit for Tabular data file ingests */ - /* (can be set separately for specific ingestable formats; in which - case the actual stored option will be TabularIngestSizeLimit:{FORMAT_NAME} - where {FORMAT_NAME} is the format identification tag returned by the - getFormatName() method in the format-specific plugin; "sav" for the - SPSS/sav format, "RData" for R, etc. - for example: :TabularIngestSizeLimit:RData */ - TabularIngestSizeLimit, - /* Validate physical files in the dataset when publishing, if the dataset size less than the threshold limit */ - DatasetChecksumValidationSizeLimit, - /* Validate physical files in the dataset when publishing, if the datafile size less than the threshold limit */ - DataFileChecksumValidationSizeLimit, - FilePIDsEnabled, - /** - * If defined, this is the URL of the zipping service outside - * the main Application Service where zip downloads should be directed - * instead of /api/access/datafiles/ - */ - CustomZipDownloadServiceUrl, - - /** - * The URL for the DvWebLoader tool (see github.com/gdcc/dvwebloader for details) - */ - WebloaderUrl, - - - /** - * A comma-separated list of CategoryName in the desired order for files to be - * sorted in the file table display. If not set, files will be sorted - * alphabetically by default. If set, files will be sorted by these categories - * and alphabetically within each category. - */ - CategoryOrder, - /** - * True(default)/false option deciding whether ordering by folder should be applied to the - * dataset listing of datafiles. - */ - OrderByFolder, - /** - * Allows an instance admin to disable Solr search facets on the collection - * and dataset pages instantly - */ - DisableSolrFacets, - DisableSolrFacetsForGuestUsers, - DisableSolrFacetsWithoutJsession, - DisableUncheckedTypesFacet, - - /** - * When ingesting tabular data files, store the generated tab-delimited - * files *with* the variable names line up top. - */ - StoreIngestedTabularFilesWithVarHeaders, - - There is no well-defined cut-off - the speed of viewing and editing a large dataset will decrease as files are added, and, at some point, Dataverse may need more memory than is available, resulting in drastically slower response / +To configure these options, an administrator must set two JVM options for the Dataverse installation using the same process as for other configuration options: +``./asadmin create-jvm-options "-Ddataverse.files..download-redirect=true"`` +``./asadmin create-jvm-options "-Ddataverse.files..upload-redirect=true"`` +With multiple stores configured, it is possible to configure one S3 store with direct upload and/or download to support large files (in general or for specific Dataverse collections) while configuring only direct download, or no direct access for another store. The direct upload option now switches between uploading the file in one piece (up to 1 GB by default) and sending it as multiple parts. The default can be changed by setting: @@ -365,7 +39,7 @@ At present, one potential drawback for direct-upload is that files are only part .. _s3-direct-upload-features-disabled: Features that are Disabled if S3 Direct Upload is Enabled -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The following features are disabled when S3 direct upload is enabled. @@ -378,7 +52,7 @@ The following features are disabled when S3 direct upload is enabled. .. _cors-s3-bucket: Allow CORS for S3 Buckets -^^^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~~~~ **IMPORTANT:** One additional step that is required to enable direct uploads via a Dataverse installation and for direct download to work with previewers and direct upload to work with dvwebloader (:ref:`folder-upload`) is to allow cross site (CORS) requests on your S3 store. The example below shows how to enable CORS rules (to support upload and download) on a bucket using the AWS CLI command line tool. Note that you may want to limit the AllowedOrigins and/or AllowedHeaders further. https://github.com/gdcc/dataverse-previewers/wiki/Using-Previewers-with-download-redirects-from-S3 has some additional information about doing this. @@ -411,14 +85,12 @@ Alternatively, you can enable CORS using the AWS S3 web interface, using json-en .. _s3-tags-and-direct-upload: S3 Tags and Direct Upload -^^^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~~~~ Since the direct upload mechanism creates the final file rather than an intermediate temporary file, user actions, such as neither saving or canceling an upload session before closing the browser page, can leave an abandoned file in the store. The direct upload mechanism attempts to use S3 tags to aid in identifying/removing such files. Upon upload, files are given a "dv-state":"temp" tag which is removed when the dataset changes are saved and new files are added in the Dataverse installation. Note that not all S3 implementations support tags. Minio, for example, does not. With such stores, direct upload may not work and you might need to disable tagging. For details, see :ref:`s3-tagging` in the Installation Guide. Trusted Remote Storage with the ``remote`` Store Type -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -**Best for:** Very large files that should remain in their original location +----------------------------------------------------- For very large, and/or very sensitive data, it may not make sense to transfer or copy files to Dataverse at all. The experimental ``remote`` store type in the Dataverse software now supports this use case. @@ -479,7 +151,7 @@ To configure the options mentioned above, an administrator must set two JVM opti .. _globus-support: Globus File Transfer -~~~~~~~~~~~~~~~~~~~ +-------------------- Note: Globus file transfer is still experimental but feedback is welcome! See :ref:`support`. @@ -518,85 +190,3 @@ An overview of the control and data transfer interactions between components was See also :ref:`Globus settings <:GlobusSettings>`. An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. Due to its experimental nature it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag (see :ref:`feature-flags`) and the JVM option :ref:`dataverse.files.globus-monitoring-server`. - -Handling High Volume of Files ----------------------------- - -When dealing with datasets containing thousands or millions of files, different challenges arise beyond just file size. - -Performance Considerations -~~~~~~~~~~~~~~~~~~~~~~~~~ - -**Database Impact** - -Each file in Dataverse requires database entries for metadata, permissions, and other attributes. When dealing with very large numbers of files: - -* Database query performance may degrade -* Indexing operations take longer -* Dataset versioning becomes more resource-intensive - -**Recommended Approaches** - -For datasets with extremely high file counts (10,000+), consider these strategies: - -1. **Use hierarchical organization** - Group files into logical directories to improve navigation -2. **Batch operations** - Use the API for batch uploads rather than the UI -3. **Consider file bundling** - Where appropriate, bundle related small files into archives -4. **Use Globus for bulk transfers** - Globus is optimized for handling large numbers of files - -API-Based Batch Operations -~~~~~~~~~~~~~~~~~~~~~~~~~ - -For programmatically handling large numbers of files, Dataverse provides API endpoints that support batch operations: - -* The `/api/datasets/:id/addFiles` endpoint allows adding multiple files in a single request -* The `/api/datasets/:id/deleteFiles` endpoint allows removing multiple files at once - -See the :doc:`/api/native-api` documentation for details on these endpoints. - -Monitoring and Maintenance -~~~~~~~~~~~~~~~~~~~~~~~~~ - -When working with high file volumes: - -* Monitor database performance regularly -* Consider implementing scheduled maintenance windows for index optimization -* Use the Dataverse metrics API to track system performance - -Storage Strategy Recommendations -------------------------------- - -Based on both file size and volume considerations, here are some general recommendations: - -1. **For research projects with moderate data (< 2GB files, < 1000 files):** - * Default Dataverse storage is sufficient - -2. **For projects with large files but moderate file counts:** - * Configure S3 direct upload/download - * Set appropriate ingest size limits - -3. **For projects with very large files or sensitive data that should remain in place:** - * Use the remote storage option - * Consider implementing access controls at the remote storage level - -4. **For high-performance computing environments or very large datasets:** - * Implement Globus transfer - * Use batch operations via API - * Consider custom workflows for dataset creation and management - -5. **For datasets with millions of small files:** - * Consider file bundling where appropriate - * Use Globus for transfer - * Implement hierarchical organization - -Performance Tuning ------------------ - -Regardless of which storage solution you choose, consider these performance tuning options: - -* Increase JVM heap size for Dataverse application -* Optimize database indexes for file metadata tables -* Configure appropriate timeouts for large file transfers -* Consider dedicated storage nodes for high-volume installations - -For detailed performance tuning recommendations, see the :doc:`/installation/config` guide. From 158ea2aceadf29c613abf18615a69ad9ef858bdf Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 26 Sep 2025 15:11:53 -0400 Subject: [PATCH 307/634] fix lists /errs in rst --- .../source/admin/big-data-administration.rst | 57 ++++++++++--------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/doc/sphinx-guides/source/admin/big-data-administration.rst b/doc/sphinx-guides/source/admin/big-data-administration.rst index dba872aed5e..c28d9d600b3 100644 --- a/doc/sphinx-guides/source/admin/big-data-administration.rst +++ b/doc/sphinx-guides/source/admin/big-data-administration.rst @@ -36,9 +36,9 @@ version of the file. Benefits: - This option requires no external services and can potentially handle files into the gigabyte (GB) size range. For smaller institutions, -and in disciplines where datasets do not have more than a few hundred files and files are not too large, this can be the simplest option. + and in disciplines where datasets do not have more than a few hundred files and files are not too large, this can be the simplest option. - Unzipping of zip archives can be simpler for users than having to upload many individual files and was, at one time, the only way to -preserve file path names when uploading. + preserve file path names when uploading. Challenges: In general file storage is not a good option for larger data sizes - both in terms of file size and number of files. Contributing factors include: @@ -65,18 +65,18 @@ the S3 store. In this configuration, files are never stored on the Dataverse ser Benefits: S3 offers several advantages over file storage which : - Scalability: S3 is designed to handle large amounts of data. It can handle individual files up to several TB in size. -Because S3 supports breaking files into pieces, Dataverse can transfer a file in pieces (several in parallel, potentially thousands of pieces per file) making transfers faster -and more robust (a failure requires only resending the failed piece). It may also be the case that users will have a faster network connection to the S3 store -(e.g. in a commercial cloud or High Performance Computing center) than they do to the Dataverse server, reducing transfer time. + Because S3 supports breaking files into pieces, Dataverse can transfer a file in pieces (several in parallel, potentially thousands of pieces per file) making transfers faster + and more robust (a failure requires only resending the failed piece). It may also be the case that users will have a faster network connection to the S3 store + (e.g. in a commercial cloud or High Performance Computing center) than they do to the Dataverse server, reducing transfer time. - High Availability: S3 provides redundancy beyond what is available with a single disk (valuable for preservation, potentially reducing the need to perform data integrity checks). Challenges: - One additional step that is required to enable direct uploads via a Dataverse installation and for direct download to work with previewers and direct upload to work with dvwebloader (:ref:`folder-upload`) is to allow cross site (CORS) requests on your S3 store. - Cost: S3 offers a pricing model that allows you to pay for the storage and transfer of data based on current usage (versus long term demand) but commercial -providers charge more per TB than the equivalent cost of a local disk (though commercial S3 storage is cheaper than commercial file storage). -There can also be egress and other charges. Overall, S3 storage is generally more expensive than local file storage but cheaper than cloud file storage. -Running a local S3 storage or leveraging an institutional service can further reduce costs. + providers charge more per TB than the equivalent cost of a local disk (though commercial S3 storage is cheaper than commercial file storage). + There can also be egress and other charges. Overall, S3 storage is generally more expensive than local file storage but cheaper than cloud file storage. + Running a local S3 storage or leveraging an institutional service can further reduce costs. - Direct upload via S3 is a multi-step process: Dataverse provides URLs for the uploads, the user's browser or other app uses the URLs to transfer files to the S3 store, possibly in many pieces per file, and finally, Dataverse is told that one or more files are in place and should be added to the dataset. If the last step fails, or if all parts of a file cannot be transferred, orphaned files or parts of files can be left on S3. These files are not accessible via Dataverse but do use space (for which there is a monetary cost) @@ -87,24 +87,26 @@ Other Considerations - S3 Storage without direct upload/download provides minimal benefits with Dataverse as files still pass through the server, files are still uploaded as a single HTTP/HTTPS stream, and temporary storage is still used. - While not having files unzipped can be confusing to users who are used to it from using Dataverse with file storage, there are ways to minimize the impact. -For example, Dataverse can be configured to use a 'Zip File Previewer' that allows users to see the contents of a zip file and even download individual files from within it. -For users who still want their data stored as individual files with their relative folder paths, Dataverse can be configured with the 'DVWebloader' which allows users to select an entire folder tree of files and -upload them, with their relative paths intact, to Dataverse. (DVWebloader can only be used with S3/direct upload, but it is much more efficient with many files than using the -standard upload interface in Dataverse (which also does not retain path information)). + For example, Dataverse can be configured to use a 'Zip File Previewer' that allows users to see the contents of a zip file and even download individual files from within it. + For users who still want their data stored as individual files with their relative folder paths, Dataverse can be configured with the 'DVWebloader' which allows users to select an entire folder tree of files and + upload them, with their relative paths intact, to Dataverse. (DVWebloader can only be used with S3/direct upload, but it is much more efficient with many files than using the + standard upload interface in Dataverse (which also does not retain path information)). - The following features are disabled when S3 direct upload is enabled. - Unzipping of zip files. (See :ref:`compressed-files`.) - Detection of file type based on JHOVE and custom code that reads the first few bytes except for the refinement of Stata file types to include the version. (See :ref:`redetect-file-type`.) - Extraction of metadata from FITS files. (See :ref:`fits`.) - Creation of NcML auxiliary files (See :ref:`netcdf-and-hdf5`.) - Extraction of a geospatial bounding box from NetCDF and HDF5 files (see :ref:`netcdf-and-hdf5`) unless :ref:`dataverse.netcdf.geo-extract-s3-direct-upload` is set to true. + - Using direct upload stops Dataverse from inspecting the file bytes to determine the MIME type (with one exception - Stata files). Dataverse will still look at the file name and extension to determine the MIME type. - To perform the 'ingest' processing, Dataverse currently has to copy the file to local storage, somewhat negating the benefit of sending data directly to S3. To manage larger files, one can set a per-store -ingest size limit (which can be 0 bytes) to stop ingest or limit it to smaller files. + ingest size limit (which can be 0 bytes) to stop ingest or limit it to smaller files. - Dataverse's mechanism for downloading a whole dataset or multiple selected files involves zipping those files together. Even When using S3 with direct upload/download, -the file bytes are transferred to the Dataverse server as part of the zipping process. There are ways to reduce the performance impact of this: + the file bytes are transferred to the Dataverse server as part of the zipping process. There are ways to reduce the performance impact of this: - There is a 'ZipDownloader' app that can be run separate from Dataverse to handle the zipping process. - Dataverse has a :ZipDownloadLimit that can be used to limit the amount of data that can be zipped. If a dataset is larger than this limit, Dataverse will only add some of the files to the zip and list others in the included manifest file. - There are tools such as the Dataverse Dataset Downloader (https://github.com/gdcc/dataverse-recipes/tree/main/shell/download#dataverse-dataset-downloader) that can be used to download all off the files individually. This avoids sending any of the files to/through the Dataverse server when S3 direct download is enabled. + - Dataverse leverages S3 features that are not implemented by all servers and has several configuration options geared towards handling variations between servers. Site admins should be sure to test with their preferred S3 implementation. - The part-size used when directly transferring files to S3 is configurable (at AWS, from 5 MiB to 5GiB). The default in Dataverse is 1 GiB (1024^3). If the primary use case is with smaller files than that, decreasing the part size may improve upload speeds. @@ -145,9 +147,9 @@ Challenges: - Currently, remote files can only be added via the API. (This may be addressed in future versions). - Remote files can only be added after the dataset is created in the UI (and therefore has an id and PID for use with the API). However, the UI will still allow upload of files to the base store (at dataset creation and when editing), which could be confusing. - As Dataverse is relying on the remote service to main the integrity and availability of the files, it is likely that the Dataverse site admin will want to have a formal agreement with the remote service -operator about their policies. + operator about their policies. - Site admins need to consider carefully how to configure file size limits, ingest size limits, etc. on the remote store and it's base store, and whether the remote store is public-only, and whether files there can be read by Dataverse to assure the -requirements of a specific use case(s) are addressed. + requirements of a specific use case(s) are addressed. - The current remote store implementation will not prevent you from providing a relative URL that results in a 404 when resolved. (I.e. if you make a typo). You should check to make sure the file exists at the location you specify - by trying to download in Dataverse, by checking to see that Dataverse was able to get the file size (which it does by doing a HEAD call to that location), or just manually trying the URL in your browser. - For large files, direct-download should always be used with a remote store. (Otherwise the Dataverse will be involved in the download.) - When multiple files are selected for download, Dataverse will try to include remote files in the zip file being created (up to the max zip size limit) which is inefficient, and will not be able to include remote files that are inaccessible (possibly confusing). @@ -181,24 +183,24 @@ Challenges: - For users not familiar with Globus, managing transfers can be confusing. For the non-S3 options, users cannot just download files - they must have access to a destination Globus endpoint and have a Globus account. Globus does provide free accounts and a free 'Globus Personal Connect' service which installed on any machine to allow transfers to/from it. - Globus transfers are not enabled at dataset-creation time. Once the draft version is created, users can initiate Globus transfers to upload files from remote endpoints. - For Dataverse-managed endpoints, a community-developed `dataverse-globus `_ app must be installed and configured in the Dataverse instance. -This app manages granting and revoking access for users to upload/download files from Dataverse and handles the translation between Dataverse's internal file naming/organization to that see by the user. + This app manages granting and revoking access for users to upload/download files from Dataverse and handles the translation between Dataverse's internal file naming/organization to that see by the user. - Users familiar with Globus sometimes expect to be able to find the Dataverse endpoint in Globus' online service and download files from there. Due to the fact that Dataverse is managing permissions and handling file naming, this doesn't work. - Due to differences between Dataverse's and Globus's access control models, Dataverse cannot enforce per-file-access restrictions - restriction can only be done today at the level of providing access to all files in a dataset. -Globus stores can be defined as public to disable Dataverse's ability to restrict and embargo files in that store. If the store is configured to support restriction and embargo, -Dataverse and it's Dataverse-Globus app will limit users to downloading only the files they have been granted access to, but a technically knowledgeable user could access other files in the same dataset if they are give access to one. -(Data depositors would need to be aware of this limitation and could be guided to restrict all files/only grant access to all dataset files in Globus as a work-around). + Globus stores can be defined as public to disable Dataverse's ability to restrict and embargo files in that store. If the store is configured to support restriction and embargo, + Dataverse and it's Dataverse-Globus app will limit users to downloading only the files they have been granted access to, but a technically knowledgeable user could access other files in the same dataset if they are give access to one. + (Data depositors would need to be aware of this limitation and could be guided to restrict all files/only grant access to all dataset files in Globus as a work-around). - Dataverse-managed endpoints must be Globus 'guest collections' hosted on either a file-system-based endpoint or an S3-based endpoint (the latter requires use of the Globus -S3 connector which requires a paid Globus subscription at the host institution). In either case, Dataverse is configured with the Globus credentials of a user account that can manage the endpoint. -Users will need their own Globus account, which can be obtained via their institution or directly from Globus (at no cost). + S3 connector which requires a paid Globus subscription at the host institution). In either case, Dataverse is configured with the Globus credentials of a user account that can manage the endpoint. + Users will need their own Globus account, which can be obtained via their institution or directly from Globus (at no cost). - With the file-system endpoint, Dataverse does not currently have access to the file contents. Thus, functionality related to ingest, previews, fixity hash validation, etc. are not available. -(Using the S3-based endpoint, Dataverse has access via S3 and all functionality normally associated with direct uploads to S3 is available. In this case admins should be sure to set the maximum size for ingest and avoid requiring hash validation at publication, etc.) + (Using the S3-based endpoint, Dataverse has access via S3 and all functionality normally associated with direct uploads to S3 is available. In this case admins should be sure to set the maximum size for ingest and avoid requiring hash validation at publication, etc.) - For the reference use case, Dataverse must be configured with a list of allowed endpoint/base paths from which files may be referenced. In this case, since Dataverse is not accessing the remote endpoint itself, it does not need Globus credentials. -Users will also need a Globus account in this case, and the remote endpoint must be configured to allow them access (i.e. be publicly readable, or potentially involving some out-of-band mechanism to request access (that could be described, for example, in the dataset's Terms of Use and Access). + Users will also need a Globus account in this case, and the remote endpoint must be configured to allow them access (i.e. be publicly readable, or potentially involving some out-of-band mechanism to request access (that could be described, for example, in the dataset's Terms of Use and Access). - As with remote stores, files can only be added in the Globus reference case via the Dataverse API. - While Globus itself can handle many (millions of) files of any size, Dataverse cannot handle more than thousands of files per dataset (at best) and some Globus endpoints may have limits on file sizes - both maximums and minimums (e.g. for tape storage where small files are inefficient). -Users will need to be made aware of these limitations and the possibilities for managing them (e.g. by aggregating multiple files in a single larger file, or storing smaller files in the base-store via the normal Dataverse upload UI). + Users will need to be made aware of these limitations and the possibilities for managing them (e.g. by aggregating multiple files in a single larger file, or storing smaller files in the base-store via the normal Dataverse upload UI). - There is currently (a bug)[https://github.com/gdcc/dataverse-globus/issues/2] that won't allow users to transfer files from/to endpoints where they do not have permission to list the overall file tree (i.e. an institution manages /institution_name but the user only has access to /institution_name/my_dir.) -Until that is fixed, a work-around is to first transfer data to an endpoint without this restriction. + Until that is fixed, a work-around is to first transfer data to an endpoint without this restriction. - An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. While it is now the recommended option, it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag (see :ref:`feature-flags`) and the JVM option :ref:`dataverse.files.globus-monitoring-server`. More details of the setup required to enable Globus is described in the `Community Dataverse-Globus Setup and Configuration document `_ and the references therein. @@ -209,7 +211,7 @@ See also :ref:`Globus settings <:GlobusSettings>`. Storage Strategy Recommendations -------------------------------- +-------------------------------- Based on both file size and volume considerations, here are some general recommendations: @@ -254,7 +256,6 @@ There are a number of settings to limit how many files can be uploaded: - ZipUploadFilesLimit - the maximum number of files allowed in an uploaded zip file - only relevant for file stores and S3 when direct upload is not used. - MultipleUploadFilesLimit - the number of files the GUI user is allowed to upload in one batch, via drag-and-drop, or through the file select dialog - MaxFileUploadSizeInBytes - limit the size of files that can be uploaded - - UseStorageQuotas - once enabled, super users can set per-collection quotas (in bytes) to limit the aggregate size of all files in the collection - dataverse.files.default-dataset-file-count-limit - directly limits the number of files per dataset, can be changed per dataset via API (by super users) From f3b758927b02babed7c347cd522065eb68dc773f Mon Sep 17 00:00:00 2001 From: Maham Maham <116267645+Maham802@users.noreply.github.com> Date: Fri, 26 Sep 2025 21:33:41 +0200 Subject: [PATCH 308/634] fix: update dataverse-ansible links in deployment.rst to develop branch --- doc/sphinx-guides/source/developers/deployment.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/developers/deployment.rst b/doc/sphinx-guides/source/developers/deployment.rst index d3e0c33a8f2..ec9929136b7 100755 --- a/doc/sphinx-guides/source/developers/deployment.rst +++ b/doc/sphinx-guides/source/developers/deployment.rst @@ -78,7 +78,7 @@ Amazon offers instructions on using an IAM role to grant permissions to applicat Configure Ansible File (Optional) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In order to configure Dataverse installation settings such as the password of the dataverseAdmin user, download https://raw.githubusercontent.com/gdcc/dataverse-ansible/master/defaults/main.yml and edit the file to your liking. +In order to configure Dataverse installation settings such as the password of the dataverseAdmin user, download https://raw.githubusercontent.com/gdcc/dataverse-ansible/develop/defaults/main.yml and edit the file to your liking. You can skip this step if you're fine with the values in the "main.yml" file in the link above. @@ -89,7 +89,7 @@ Once you have done the configuration above, you are ready to try running the "ec Download `ec2-create-instance.sh`_ and put it somewhere reasonable. For the purpose of these instructions we'll assume it's in the "Downloads" directory in your home directory. -.. _ec2-create-instance.sh: https://raw.githubusercontent.com/gdcc/dataverse-ansible/master/ec2/ec2-create-instance.sh +.. _ec2-create-instance.sh: https://raw.githubusercontent.com/gdcc/dataverse-ansible/develop/ec2/ec2-create-instance.sh To run the script, you can make it executable (``chmod 755 ec2-create-instance.sh``) or run it with bash, like this with ``-h`` as an argument to print the help: From 75de0ea4e3fc0c765b54854e71ba6bf5fa132f67 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 26 Sep 2025 15:43:32 -0400 Subject: [PATCH 309/634] fix strategy table --- doc/sphinx-guides/source/admin/big-data-administration.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/admin/big-data-administration.rst b/doc/sphinx-guides/source/admin/big-data-administration.rst index c28d9d600b3..fd2316525ab 100644 --- a/doc/sphinx-guides/source/admin/big-data-administration.rst +++ b/doc/sphinx-guides/source/admin/big-data-administration.rst @@ -216,22 +216,27 @@ Storage Strategy Recommendations Based on both file size and volume considerations, here are some general recommendations: 1. **For research projects with moderate data (< 2GB files, < 100s of files/dataset):** + * Default File Store is sufficient * Consider setting file count and size limits (see below) 2. **For projects with larger files (GBs to TBs) and/or more files per dataset (100s to 1000s):** - * Configure an S3 store witj direct upload/download + + * Configure an S3 store with direct upload/download * Set appropriate ingest size limits 3. **For projects with very large files or sensitive data that should remain in place:** + * Use the Remote Store 4. **For high-performance computing environments or very large datasets (TBs+):** + * Use a Globus Store * Work with users to size files appropriate to the underlying storage * Consider Globus over S3 when normal upload/download options (via the UI/API) are desired along with Globus transfer 5. **For Petascale datasets, or extreme numbers of files:** + * Consider a Remote Store and referencing a single URL/Globus endpoint for the entire dataset From 76f4e5260683685289db76491d5e7071659561a9 Mon Sep 17 00:00:00 2001 From: Maham Maham <116267645+Maham802@users.noreply.github.com> Date: Fri, 26 Sep 2025 21:54:02 +0200 Subject: [PATCH 310/634] fix(docs): update dataverse-ansible links in testing.rst to use develop branch --- doc/sphinx-guides/source/developers/testing.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index 8f9ce645a2e..bd91ce27d54 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -128,7 +128,7 @@ You might find studying the following test classes helpful in writing tests for - DeletePrivateUrlCommandTest.java - GetPrivateUrlCommandTest.java -In addition, there is a writeup on "The Testable Command" at https://github.com/IQSS/dataverse/blob/develop/doc/theTestableCommand/TheTestableCommand.md . +In addition, there is a writeup on "The Testable Command" at https://github.com/IQSS/dataverse/blob/master/doc/theTestableCommand/TheTestableCommand.md . Running Non-Essential (Excluded) Unit Tests ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -169,12 +169,12 @@ different people. For our purposes, an integration test can have two flavors: Running the Full API Test Suite Using EC2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -**Prerequisite:** To run the API test suite in an EC2 instance you should first follow the steps in the :doc:`deployment` section to get set up with the AWS binary to launch EC2 instances. If you're here because you just want to spin up a branch, you'll still want to follow the AWS deployment setup steps, but may find the `ec2-create README.md `_ Quick Start section helpful. +**Prerequisite:** To run the API test suite in an EC2 instance you should first follow the steps in the :doc:`deployment` section to get set up with the AWS binary to launch EC2 instances. If you're here because you just want to spin up a branch, you'll still want to follow the AWS deployment setup steps, but may find the `ec2-create README.md `_ Quick Start section helpful. You may always retrieve a current copy of the ec2-create-instance.sh script and accompanying group_var.yml file from the `dataverse-ansible repo `_. Since we want to run the test suite, let's grab the group_vars used by Jenkins: -- `ec2-create-instance.sh `_ -- `jenkins.yml `_ +- `ec2-create-instance.sh `_ +- `jenkins.yml `_ Edit ``jenkins.yml`` to set the desired GitHub repo and branch, and to adjust any other options to meet your needs: @@ -184,7 +184,7 @@ Edit ``jenkins.yml`` to set the desired GitHub repo and branch, and to adjust an - ``dataverse.unittests.enabled: true`` - ``dataverse.sampledata.enabled: true`` -If you wish, you may pass the script a ``-l`` flag with a local relative path in which the script will `copy various logs `_ at the end of the test suite for your review. +If you wish, you may pass the script a ``-l`` flag with a local relative path in which the script will `copy various logs `_ at the end of the test suite for your review. Finally, run the script: From 82697886a262b088d4c5aa7eba56b819329d3961 Mon Sep 17 00:00:00 2001 From: Maham Maham <116267645+Maham802@users.noreply.github.com> Date: Fri, 26 Sep 2025 22:00:49 +0200 Subject: [PATCH 311/634] fix: removed line number --- doc/sphinx-guides/source/developers/testing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index bd91ce27d54..f84a7cf1ac7 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -184,7 +184,7 @@ Edit ``jenkins.yml`` to set the desired GitHub repo and branch, and to adjust an - ``dataverse.unittests.enabled: true`` - ``dataverse.sampledata.enabled: true`` -If you wish, you may pass the script a ``-l`` flag with a local relative path in which the script will `copy various logs `_ at the end of the test suite for your review. +If you wish, you may pass the script a ``-l`` flag with a local relative path in which the script will `copy various logs `_ at the end of the test suite for your review. Finally, run the script: From 8a51ebc9b02177b89b2014cc52876432edaeb97a Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 26 Sep 2025 16:05:40 -0400 Subject: [PATCH 312/634] #11772 fix parser and tests --- .../harvard/iq/dataverse/api/Datasets.java | 1 - .../impl/UpdateDatasetTermsOfUseCommand.java | 5 --- .../iq/dataverse/util/json/JsonParser.java | 12 +++++-- .../harvard/iq/dataverse/api/DatasetsIT.java | 35 +++++++++++++++++++ .../json/update-dataset-terms-access.json | 6 ++-- 5 files changed, 48 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 355db7c8c60..9a8c4ab4d2a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1172,7 +1172,6 @@ public Response editVersionTermsOfUse(@Context ContainerRequestContext crc, Stri toua.setDatasetVersion(dataset.getLatestVersion()); - DatasetVersion updatedVersion = execCommand(new UpdateDatasetTermsOfUseCommand(dataset, toua, createDataverseRequest(getRequestUser(crc)))).getLatestVersion(); return ok(json(updatedVersion, true)); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommand.java index 6e7a5c29e57..74aa2ac0138 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommand.java @@ -48,11 +48,6 @@ public Dataset execute(CommandContext ctxt) throws CommandException { datasetVersion.setVersionState(DatasetVersion.VersionState.DRAFT); termsOfUseAndAccess.setDatasetVersion(datasetVersion); - System.out.print("From input...."); - System.out.print(termsOfUseAndAccess.getConfidentialityDeclaration()); - System.out.print("From DatasetVersion...."); - System.out.print(datasetVersion.getTermsOfUseAndAccess().getConfidentialityDeclaration()); - System.out.print("$$$$$$$$$$$$$$$$$$$$$$$$"); return ctxt.engine().submit(updateDatasetVersionCommand == null ? new UpdateDatasetVersionCommand(this.dataset, getRequest()) : updateDatasetVersionCommand); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index b74f852202c..a080241777e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -379,9 +379,10 @@ public static > List parseEnumsFromArray(JsonArray enumsArr } public TermsOfUseAndAccess parseTermsOfUseAndAccess(JsonObject obj) throws JsonParseException { - JsonObject terms = obj.getJsonObject("termsOfAccess"); + JsonObject terms = obj.getJsonObject("termsOfUseAndAccess"); TermsOfUseAndAccess toaa = new TermsOfUseAndAccess(); - + toaa.setFileAccessRequest(terms.getBoolean("fileAccessRequest", false)); + toaa.setTermsOfAccess(terms.getString("termsOfAccess", null)); toaa.setTermsOfUse(terms.getString("termsOfUse", null)); toaa.setConfidentialityDeclaration(terms.getString("confidentialityDeclaration", null)); toaa.setSpecialPermissions(terms.getString("specialPermissions", null)); @@ -390,6 +391,13 @@ public TermsOfUseAndAccess parseTermsOfUseAndAccess(JsonObject obj) throws JsonP toaa.setDepositorRequirements(terms.getString("depositorRequirements", null)); toaa.setConditions(terms.getString("conditions", null)); toaa.setDisclaimer(terms.getString("disclaimer", null)); + toaa.setDataAccessPlace(terms.getString("dataAccessPlace", null)); + toaa.setOriginalArchive(terms.getString("originalArchive", null)); + toaa.setAvailabilityStatus(terms.getString("availabilityStatus", null)); + toaa.setContactForAccess(terms.getString("contactForAccess", null)); + toaa.setSizeOfCollection(terms.getString("sizeOfCollection", null)); + toaa.setStudyCompletion(terms.getString("studyCompletion", null)); + toaa.setConfidentialityDeclaration(terms.getString("confidentialityDeclaration", null)); return toaa; } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 91196b6b9d5..bb4793961f8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -4069,6 +4069,41 @@ public void testUpdateDatasetTerms() throws IOException { updateTerms.then().assertThat() .statusCode(OK.getStatusCode()); + //Add another dataset with restricted files in order to get the access info to show up + + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + int datasetId2 = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + String datasetPersistentId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId"); + + updateTerms = UtilIT.updateDatasetTermsAndAccess(datasetPersistentId, apiToken, pathToJsonFile); + updateTerms.prettyPrint(); + updateTerms.then().assertThat() + .statusCode(OK.getStatusCode()); + + // Upload file + String pathToTestFile = "src/test/resources/images/coffeeshop.png"; + Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId2), pathToTestFile, Json.createObjectBuilder().build(), apiToken); + uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); + + String fileId = JsonPath.from(uploadResponse.body().asString()).getString("data.files[0].dataFile.id"); + + updateTerms = UtilIT.updateDatasetTermsAndAccess(datasetPersistentId, apiToken, pathToJsonFile); + updateTerms.prettyPrint(); + updateTerms.then().assertThat() + .statusCode(OK.getStatusCode()); + + // Restrict file + Response restrictFileResponse = UtilIT.restrictFile(fileId, true, apiToken); + restrictFileResponse.then().assertThat().statusCode(OK.getStatusCode()); + + // Publish dataset version + Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetPersistentId, "major", apiToken); + publishDatasetResponse.prettyPrint(); + publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode()); + + } diff --git a/src/test/resources/json/update-dataset-terms-access.json b/src/test/resources/json/update-dataset-terms-access.json index dbe6380ca56..29749cd21dd 100644 --- a/src/test/resources/json/update-dataset-terms-access.json +++ b/src/test/resources/json/update-dataset-terms-access.json @@ -1,7 +1,7 @@ { - "termsOfAccess": { - "fileAccessRequest": "true", - "termsOfAccessForRestrictedFiles": "For access to restriced files please see read me file", + "termsOfUseAndAccess": { + "fileAccessRequest": "false", + "termsOfAccess": "For access to restriced files please see read me file", "dataAccessPlace": "dataAccessPlace", "originalArchive": "originalArchive", "availabilityStatus": "availabilityStatus", From 509e640790c77e00d1617a749e0c64301cc25e3a Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 29 Sep 2025 13:56:54 +0100 Subject: [PATCH 313/634] Added: dataset display name and identifier to InApp file notifications --- .../json/InAppNotificationsJsonPrinter.java | 2 ++ .../InAppNotificationsJsonPrinterTest.java | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/InAppNotificationsJsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/InAppNotificationsJsonPrinter.java index f3c3a0383e0..241b2d44bad 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/InAppNotificationsJsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/InAppNotificationsJsonPrinter.java @@ -184,6 +184,8 @@ private void addDataFileFields(final NullSafeJsonBuilder notificationJson, final if (dataFile != null) { notificationJson.add(KEY_DATAFILE_ID, dataFile.getId()); notificationJson.add(KEY_DATAFILE_DISPLAY_NAME, dataFile.getDisplayName()); + notificationJson.add(KEY_DATASET_DISPLAY_NAME, dataFile.getOwner().getDisplayName()); + notificationJson.add(KEY_DATASET_PERSISTENT_ID, dataFile.getOwner().getIdentifier()); } else { notificationJson.add(KEY_OBJECT_DELETED, true); } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/InAppNotificationsJsonPrinterTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/InAppNotificationsJsonPrinterTest.java index 82c6125f6c1..2d55f488889 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/InAppNotificationsJsonPrinterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/InAppNotificationsJsonPrinterTest.java @@ -215,9 +215,14 @@ public void testAddFieldsByType_requestFileAccess() { when(requestor.getLastName()).thenReturn("Doe"); when(requestor.getEmail()).thenReturn("johndoe@example.com"); + Dataset dataset = mock(Dataset.class); + when(dataset.getDisplayName()).thenReturn("Test Dataset"); + when(dataset.getIdentifier()).thenReturn("Test Identifier"); + DataFile dataFile = mock(DataFile.class); when(dataFile.getId()).thenReturn(1L); when(dataFile.getDisplayName()).thenReturn("Test File"); + when(dataFile.getOwner()).thenReturn(dataset); when(dataFileService.find(1L)).thenReturn(dataFile); sut.addFieldsByType(notificationJson, authenticatedUser, userNotification); @@ -227,6 +232,8 @@ public void testAddFieldsByType_requestFileAccess() { verify(notificationJson).add(KEY_REQUESTOR_EMAIL, "johndoe@example.com"); verify(notificationJson).add(KEY_DATAFILE_ID, Long.valueOf("1")); verify(notificationJson).add(KEY_DATAFILE_DISPLAY_NAME, "Test File"); + verify(notificationJson).add(KEY_DATASET_DISPLAY_NAME, "Test Dataset"); + verify(notificationJson).add(KEY_DATASET_PERSISTENT_ID, "Test Identifier"); } @Test @@ -235,9 +242,14 @@ public void testAddFieldsByType_requestFileAccess_nullRequestor() { userNotification.setObjectId(1L); userNotification.setRequestor(null); + Dataset dataset = mock(Dataset.class); + when(dataset.getDisplayName()).thenReturn("Test Dataset"); + when(dataset.getIdentifier()).thenReturn("Test Identifier"); + DataFile dataFile = mock(DataFile.class); when(dataFile.getId()).thenReturn(1L); when(dataFile.getDisplayName()).thenReturn("Test File"); + when(dataFile.getOwner()).thenReturn(dataset); when(dataFileService.find(1L)).thenReturn(dataFile); sut.addFieldsByType(notificationJson, authenticatedUser, userNotification); @@ -247,6 +259,8 @@ public void testAddFieldsByType_requestFileAccess_nullRequestor() { verify(notificationJson, never()).add(eq(KEY_REQUESTOR_EMAIL), anyString()); verify(notificationJson).add(KEY_DATAFILE_ID, Long.valueOf("1")); verify(notificationJson).add(KEY_DATAFILE_DISPLAY_NAME, "Test File"); + verify(notificationJson).add(KEY_DATASET_DISPLAY_NAME, "Test Dataset"); + verify(notificationJson).add(KEY_DATASET_PERSISTENT_ID, "Test Identifier"); } @Test @@ -254,9 +268,14 @@ public void testAddFieldsByType_grantFileAccess() { userNotification.setType(UserNotification.Type.GRANTFILEACCESS); userNotification.setObjectId(1L); + Dataset dataset = mock(Dataset.class); + when(dataset.getDisplayName()).thenReturn("Test Dataset"); + when(dataset.getIdentifier()).thenReturn("Test Identifier"); + DataFile dataFile = mock(DataFile.class); when(dataFile.getId()).thenReturn(1L); when(dataFile.getDisplayName()).thenReturn("Granted File"); + when(dataFile.getOwner()).thenReturn(dataset); when(dataFileService.find(1L)).thenReturn(dataFile); sut.addFieldsByType(notificationJson, authenticatedUser, userNotification); @@ -264,6 +283,8 @@ public void testAddFieldsByType_grantFileAccess() { verify(notificationJson).add(KEY_DATAFILE_ID, Long.valueOf("1")); verify(notificationJson).add(KEY_DATAFILE_DISPLAY_NAME, "Granted File"); verify(notificationJson, never()).add(eq(KEY_REQUESTOR_FIRST_NAME), anyString()); + verify(notificationJson).add(KEY_DATASET_DISPLAY_NAME, "Test Dataset"); + verify(notificationJson).add(KEY_DATASET_PERSISTENT_ID, "Test Identifier"); } @Test From 7c6567ba8a68f24cfcb0001abc5b3bdcea4836fb Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 29 Sep 2025 14:28:18 +0100 Subject: [PATCH 314/634] Changed: DATASETMENTIONED InApp notification producing json if additional info is a json string --- .../json/InAppNotificationsJsonPrinter.java | 27 +++- .../InAppNotificationsJsonPrinterTest.java | 121 +++++++++++++++++- 2 files changed, 144 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/InAppNotificationsJsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/InAppNotificationsJsonPrinter.java index 241b2d44bad..7ae7ec8533d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/InAppNotificationsJsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/InAppNotificationsJsonPrinter.java @@ -4,8 +4,15 @@ import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.util.SystemConfig; + import jakarta.ejb.EJB; import jakarta.ejb.Stateless; +import jakarta.json.Json; +import jakarta.json.JsonException; +import jakarta.json.JsonReader; +import jakarta.json.JsonValue; + +import java.io.StringReader; import static edu.harvard.iq.dataverse.dataset.DatasetUtil.getLocaleCurationStatusLabel; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.jsonRoleAssignments; @@ -262,6 +269,24 @@ private void addIngestFields(final NullSafeJsonBuilder notificationJson, final U private void addDatasetMentionedFields(final NullSafeJsonBuilder notificationJson, final UserNotification userNotification) { addDatasetFields(notificationJson, userNotification); - notificationJson.add(KEY_ADDITIONAL_INFO, userNotification.getAdditionalInfo()); + + final String additionalInfo = userNotification.getAdditionalInfo(); + + if (additionalInfo != null && !additionalInfo.isEmpty()) { + try (StringReader stringReader = new StringReader(additionalInfo); + JsonReader jsonReader = Json.createReader(stringReader)) { + + // Try to parse the string into a JSON value + JsonValue additionalInfoJson = jsonReader.readValue(); + + // If successful, add the parsed JSON value. + notificationJson.add(KEY_ADDITIONAL_INFO, additionalInfoJson); + + } catch (JsonException e) { + // If parsing fails, it's not a valid JSON string. + // Fall back to adding it as a simple string. + notificationJson.add(KEY_ADDITIONAL_INFO, additionalInfo); + } + } } } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/InAppNotificationsJsonPrinterTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/InAppNotificationsJsonPrinterTest.java index 2d55f488889..054b61b3209 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/InAppNotificationsJsonPrinterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/InAppNotificationsJsonPrinterTest.java @@ -5,18 +5,23 @@ import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; import edu.harvard.iq.dataverse.util.SystemConfig; + import jakarta.json.JsonArrayBuilder; +import jakarta.json.JsonObject; +import jakarta.json.JsonValue; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.MockedStatic; + +import org.mockito.*; import org.mockito.junit.jupiter.MockitoExtension; import java.util.Collections; +import static org.junit.jupiter.api.Assertions.assertEquals; import static edu.harvard.iq.dataverse.util.json.InAppNotificationsJsonPrinter.*; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.*; @@ -47,6 +52,9 @@ public class InAppNotificationsJsonPrinterTest { @Mock private AuthenticatedUser requestor; + @Captor + private ArgumentCaptor jsonValueCaptor; + private UserNotification userNotification; private final GlobalId testGlobalId = new GlobalId(AbstractDOIProvider.DOI_PROTOCOL, "10.5072", "FK2/BYM3IW", "/", AbstractDOIProvider.DOI_RESOLVER_URL, null); @@ -505,6 +513,113 @@ public void testAddFieldsByType_ingestCompleted_objectDeleted() { verify(notificationJson).add(KEY_GUIDES_SECTION_PATH, GUIDES_SECTION_PATH_DATASET_MANAGEMENT_TABULAR_FILES_HTML); } + @Test + public void testAddFieldsByType_datasetMentioned_withJsonString() { + // Arrange + userNotification.setType(UserNotification.Type.DATASETMENTIONED); + userNotification.setObjectId(1L); + String jsonInfo = "{\"key\":\"value\", \"number\":123}"; + userNotification.setAdditionalInfo(jsonInfo); + + Dataset dataset = mock(Dataset.class); + Dataverse owner = mock(Dataverse.class); + + when(dataset.getGlobalId()).thenReturn(testGlobalId); + when(dataset.getDisplayName()).thenReturn("Mentioned Dataset"); + when(dataset.getOwner()).thenReturn(owner); + when(datasetService.find(1L)).thenReturn(dataset); + + when(owner.getAlias()).thenReturn("ownerDv"); + when(owner.getDisplayName()).thenReturn("Owner Dataverse"); + + // Act + sut.addFieldsByType(notificationJson, authenticatedUser, userNotification); + + // Assert + // Verify standard fields are still added + verify(notificationJson).add(KEY_DATASET_PERSISTENT_ID, testGlobalId.toString()); + verify(notificationJson).add(KEY_DATASET_DISPLAY_NAME, "Mentioned Dataset"); + verify(notificationJson).add(KEY_OWNER_ALIAS, "ownerDv"); + verify(notificationJson).add(KEY_OWNER_DISPLAY_NAME, "Owner Dataverse"); + + // Capture the JsonValue passed for the additional info + verify(notificationJson).add(eq(KEY_ADDITIONAL_INFO), jsonValueCaptor.capture()); + JsonValue capturedValue = jsonValueCaptor.getValue(); + + // Assert that the parsed object is a JsonObject with the correct properties + assertInstanceOf(JsonObject.class, capturedValue, "The parsed value should be a JsonObject."); + JsonObject capturedObject = (JsonObject) capturedValue; + assertEquals("value", capturedObject.getString("key")); + assertEquals(123, capturedObject.getInt("number")); + + // Explicitly verify it was NOT added as a plain string + verify(notificationJson, never()).add(KEY_ADDITIONAL_INFO, jsonInfo); + } + + @Test + public void testAddFieldsByType_datasetMentioned_withRegularString() { + // Arrange + userNotification.setType(UserNotification.Type.DATASETMENTIONED); + userNotification.setObjectId(1L); + String regularStringInfo = "This is just a regular string, not JSON."; + userNotification.setAdditionalInfo(regularStringInfo); + + Dataset dataset = mock(Dataset.class); + Dataverse owner = mock(Dataverse.class); + + when(dataset.getGlobalId()).thenReturn(testGlobalId); + when(dataset.getDisplayName()).thenReturn("Mentioned Dataset"); + when(dataset.getOwner()).thenReturn(owner); + when(datasetService.find(1L)).thenReturn(dataset); + + when(owner.getAlias()).thenReturn("ownerDv"); + when(owner.getDisplayName()).thenReturn("Owner Dataverse"); + + // Act + sut.addFieldsByType(notificationJson, authenticatedUser, userNotification); + + // Assert + verify(notificationJson).add(KEY_DATASET_PERSISTENT_ID, testGlobalId.toString()); + verify(notificationJson).add(KEY_DATASET_DISPLAY_NAME, "Mentioned Dataset"); + verify(notificationJson).add(KEY_OWNER_ALIAS, "ownerDv"); + verify(notificationJson).add(KEY_OWNER_DISPLAY_NAME, "Owner Dataverse"); + + // Verify it falls back to adding a plain String + verify(notificationJson).add(KEY_ADDITIONAL_INFO, regularStringInfo); + } + + @Test + public void testAddFieldsByType_datasetMentioned_withNullInfo() { + // Arrange + userNotification.setType(UserNotification.Type.DATASETMENTIONED); + userNotification.setObjectId(1L); + userNotification.setAdditionalInfo(null); // Set additionalInfo to null + + Dataset dataset = mock(Dataset.class); + Dataverse owner = mock(Dataverse.class); + + when(dataset.getGlobalId()).thenReturn(testGlobalId); + when(dataset.getDisplayName()).thenReturn("Mentioned Dataset"); + when(dataset.getOwner()).thenReturn(owner); + when(datasetService.find(1L)).thenReturn(dataset); + + when(owner.getAlias()).thenReturn("ownerDv"); + when(owner.getDisplayName()).thenReturn("Owner Dataverse"); + + // Act + sut.addFieldsByType(notificationJson, authenticatedUser, userNotification); + + // Assert + verify(notificationJson).add(KEY_DATASET_PERSISTENT_ID, testGlobalId.toString()); + verify(notificationJson).add(KEY_DATASET_DISPLAY_NAME, "Mentioned Dataset"); + verify(notificationJson).add(KEY_OWNER_ALIAS, "ownerDv"); + verify(notificationJson).add(KEY_OWNER_DISPLAY_NAME, "Owner Dataverse"); + + // Verify that the additional info key is never added + verify(notificationJson, never()).add(eq(KEY_ADDITIONAL_INFO), (JsonValue) any()); + verify(notificationJson, never()).add(eq(KEY_ADDITIONAL_INFO), (String) any()); + } + @Test public void testAddFieldsByType_datasetMentioned() { userNotification.setType(UserNotification.Type.DATASETMENTIONED); From a4baae04c7e1381f4293f7a8ca3f71cea3074f99 Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 29 Sep 2025 15:47:07 +0100 Subject: [PATCH 315/634] Added: release notes for #11804 --- .../11804-notifications-api-updates.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/release-notes/11804-notifications-api-updates.md diff --git a/doc/release-notes/11804-notifications-api-updates.md b/doc/release-notes/11804-notifications-api-updates.md new file mode 100644 index 00000000000..480f7e99863 --- /dev/null +++ b/doc/release-notes/11804-notifications-api-updates.md @@ -0,0 +1,18 @@ +## Notifications API Update + +**Endpoint:** `notifications/all` + +**Enhancements:** + +* When the query parameter `inAppNotificationFormat=true` is set: + + * Notifications of types: + + * `REQUESTFILEACCESS` + * `REQUESTEDFILEACCESS` + * `GRANTFILEACCESS` + * `REJECTFILEACCESS` + + now return both the **dataset display name** and **dataset persistent identifier**. + + * Notifications of type `DATASETMENTIONED` now return a **formatted JSON** in the `additionalInfo` field when this field contains a valid persisted JSON string, instead of a raw JSON string. From 0c1a9799f81c029dd21d010e7d79899eb3985686 Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 29 Sep 2025 16:06:18 +0100 Subject: [PATCH 316/634] Added: PR and issue numbers to release notes --- doc/release-notes/11804-notifications-api-updates.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/release-notes/11804-notifications-api-updates.md b/doc/release-notes/11804-notifications-api-updates.md index 480f7e99863..deba3336269 100644 --- a/doc/release-notes/11804-notifications-api-updates.md +++ b/doc/release-notes/11804-notifications-api-updates.md @@ -16,3 +16,6 @@ now return both the **dataset display name** and **dataset persistent identifier**. * Notifications of type `DATASETMENTIONED` now return a **formatted JSON** in the `additionalInfo` field when this field contains a valid persisted JSON string, instead of a raw JSON string. + +Related issue: #11809 +Related PR: #11851 From 2d0d062167d0d32fa15e7b141ada164ec8902deb Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 29 Sep 2025 16:08:40 +0100 Subject: [PATCH 317/634] Fixed: release notes wrong issue number --- doc/release-notes/11804-notifications-api-updates.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/11804-notifications-api-updates.md b/doc/release-notes/11804-notifications-api-updates.md index deba3336269..a704442464c 100644 --- a/doc/release-notes/11804-notifications-api-updates.md +++ b/doc/release-notes/11804-notifications-api-updates.md @@ -17,5 +17,5 @@ * Notifications of type `DATASETMENTIONED` now return a **formatted JSON** in the `additionalInfo` field when this field contains a valid persisted JSON string, instead of a raw JSON string. -Related issue: #11809 +Related issue: #11804 Related PR: #11851 From 1d7e11f6fb2c2c5517677b18f32a1a71139c3784 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 29 Sep 2025 13:26:14 -0400 Subject: [PATCH 318/634] Update doc/sphinx-guides/source/developers/making-releases.rst --- doc/sphinx-guides/source/developers/making-releases.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index 0da4d9e0a72..1672cf179e6 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -34,7 +34,7 @@ Early on, make sure it's clear what type of release this is. The steps below des Ensure Issues Have Been Created ------------------------------- -We have a "create release issues" script at https://github.com/IQSS/sabo that should be run a week or so before code freeze. +We have a "create release issues" script at https://github.com/IQSS/dv-project-metrics that should be run a week or so before code freeze. For each issue that is created by the script there is likely a corresponding step in this document that has "dedicated" label on it like this: From d58ac0880b43035ee0ff7b309d7309d02e5eea95 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 29 Sep 2025 13:33:42 -0400 Subject: [PATCH 319/634] typo --- doc/sphinx-guides/source/container/app-image.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/container/app-image.rst b/doc/sphinx-guides/source/container/app-image.rst index afffeae1c0b..12f5984a66b 100644 --- a/doc/sphinx-guides/source/container/app-image.rst +++ b/doc/sphinx-guides/source/container/app-image.rst @@ -81,7 +81,7 @@ For now, stale images will be kept on Docker Hub indefinitely. | Example: :substitution-code:`|nextVersion|-noble` | Summary: Rolling tag, equivalent to ``unstable`` for current development cycle. Will roll over to the rolling production tag after a Dataverse release. - | Discussion: Perhaps you are eager to starting testing features of an upcoming version (e.g. |nextVersion|) in a staging environment. You select the :substitution-code:`|nextVersion|-noble` tag (as opposed to ``unstable``) because you want to stay on |nextVersion| rather switching to the version **after that** when a release is made (which would happen if you had selected the ``unstable`` tag). Also, when the next release comes out (|nextVersion| in this example), you would stay on the :substitution-code:`|nextVersion|-noble` tag, which is the same tag that someone would use who wants the final release of |nextVersion|. (See "Rolling Production", above.) + | Discussion: Perhaps you are eager to starting testing features of an upcoming version (e.g. |nextVersion|) in a staging environment. You select the :substitution-code:`|nextVersion|-noble` tag (as opposed to ``unstable``) because you want to stay on |nextVersion| rather than switching to the version **after that** when a release is made (which would happen if you had selected the ``unstable`` tag). Also, when the next release comes out (|nextVersion| in this example), you would stay on the :substitution-code:`|nextVersion|-noble` tag, which is the same tag that someone would use who wants the final release of |nextVersion|. (See "Rolling Production", above.) **NOTE**: In these tags for development usage, the version number will always be 1 minor version ahead of existing Dataverse releases. Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. From f72e10dd83daf5158d5769c4a2250c2cf38f0f98 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 29 Sep 2025 13:33:51 -0400 Subject: [PATCH 320/634] we now use Zulip rather than Matrix --- modules/container-base/README.md | 2 +- modules/container-configbaker/README.md | 2 +- src/main/docker/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/container-base/README.md b/modules/container-base/README.md index f6854482073..7a39457b766 100644 --- a/modules/container-base/README.md +++ b/modules/container-base/README.md @@ -25,7 +25,7 @@ provides in-depth information about content, building, tuning and so on for this **Where to get help and ask questions:** IQSS will not offer support on how to deploy or run it. Please reach out to the community for help on using it. -You can join the Community Chat on Matrix at https://chat.dataverse.org and https://groups.google.com/g/dataverse-community +You can join the Community Chat at https://chat.dataverse.org and https://groups.google.com/g/dataverse-community to ask for help and guidance. ## Supported Image Tags diff --git a/modules/container-configbaker/README.md b/modules/container-configbaker/README.md index 75862ee0809..8b68e3db692 100644 --- a/modules/container-configbaker/README.md +++ b/modules/container-configbaker/README.md @@ -19,7 +19,7 @@ provides information about this image. **Where to get help and ask questions:** IQSS will not offer support on how to deploy or run it. Please reach out to the community for help on using it. -You can join the Community Chat on Matrix at https://chat.dataverse.org and https://groups.google.com/g/dataverse-community +You can join the Community Chat at https://chat.dataverse.org and https://groups.google.com/g/dataverse-community to ask for help and guidance. ## Supported Image Tags diff --git a/src/main/docker/README.md b/src/main/docker/README.md index 48416c196ca..a32c91a810e 100644 --- a/src/main/docker/README.md +++ b/src/main/docker/README.md @@ -24,7 +24,7 @@ for more details on tunable settings, locations, etc. **Where to get help and ask questions:** IQSS will not offer support on how to deploy or run it. Please reach out to the community for help on using it. -You can join the Community Chat on Matrix at https://chat.dataverse.org and https://groups.google.com/g/dataverse-community +You can join the Community Chat at https://chat.dataverse.org and https://groups.google.com/g/dataverse-community to ask for help and guidance. ## Supported Image Tags From d4d1bfe3290ac12e84414779dd8bd21e0f63fbaf Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 29 Sep 2025 14:54:07 -0400 Subject: [PATCH 321/634] clarify milestones --- doc/sphinx-guides/source/developers/making-releases.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index 1672cf179e6..fbbc2e5d3ae 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -245,7 +245,7 @@ Add Milestone to Pull Requests and Issues Often someone is making sure that the proper milestone (e.g. 5.10.1) is being applied to pull requests and issues, but sometimes this falls between the cracks. -Check for merged pull requests that have no milestone by going to https://github.com/IQSS/dataverse/pulls and entering `is:pr is:merged no:milestone `_ as a query. If you find any, add the milestone to the pull request and any issues it closes. This includes the "merge develop into master" pull request above. +Check for merged pull requests that have no milestone by going to https://github.com/IQSS/dataverse/pulls and entering `is:pr is:merged no:milestone `_ as a query. If you find any, first check if those pull requests are against open pull requests. If so, do nothing. Otherwise, add the milestone to the pull request and any issues it closes. This includes the "merge develop into master" pull request above. (Optional) Test Docker Images ----------------------------- From bb2f6cde23f8dbff9a162e7bc59845afd00a7019 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 30 Sep 2025 11:10:22 +0100 Subject: [PATCH 322/634] Added: onlyUnread optional query parameter to getAllNotificationsForUser API endpoint --- .../dataverse/UserNotificationServiceBean.java | 17 ++++++++++++++--- .../harvard/iq/dataverse/api/Notifications.java | 6 ++++-- .../iq/dataverse/api/NotificationsIT.java | 11 ++++++++++- .../edu/harvard/iq/dataverse/api/UtilIT.java | 6 +++--- 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java index 2fb5ea35c45..167b66fb971 100644 --- a/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java @@ -18,7 +18,6 @@ import jakarta.ejb.Stateless; import jakarta.ejb.TransactionAttribute; import jakarta.ejb.TransactionAttributeType; -import jakarta.inject.Inject; import jakarta.inject.Named; import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; @@ -44,11 +43,23 @@ public class UserNotificationServiceBean { SettingsServiceBean settingsService; public List findByUser(Long userId) { - TypedQuery query = em.createQuery("select un from UserNotification un where un.user.id =:userId order by un.sendDate desc", UserNotification.class); + return findByUser(userId, false); + } + + public List findByUser(Long userId, boolean onlyUnread) { + TypedQuery query = em.createQuery( + "select un from UserNotification un " + + "where un.user.id = :userId and (:onlyUnread = false or un.readNotification = false) " + + "order by un.sendDate desc", + UserNotification.class + ); + query.setParameter("userId", userId); + query.setParameter("onlyUnread", onlyUnread); + return query.getResultList(); } - + public List findByRequestor(Long userId) { TypedQuery query = em.createQuery("select un from UserNotification un where un.requestor.id =:userId order by un.sendDate desc", UserNotification.class); query.setParameter("userId", userId); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Notifications.java b/src/main/java/edu/harvard/iq/dataverse/api/Notifications.java index eebdce5e509..c14c2edc19d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Notifications.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Notifications.java @@ -27,10 +27,12 @@ public class Notifications extends AbstractApiBean { @GET @AuthRequired @Path("/all") - public Response getAllNotificationsForUser(@Context ContainerRequestContext crc, @QueryParam("inAppNotificationFormat") boolean inAppNotificationFormat) { + public Response getAllNotificationsForUser(@Context ContainerRequestContext crc, + @QueryParam("onlyUnread") boolean onlyUnread, + @QueryParam("inAppNotificationFormat") boolean inAppNotificationFormat) { try { AuthenticatedUser authenticatedUser = getRequestAuthenticatedUserOrDie(crc); - List userNotifications = userNotificationSvc.findByUser(authenticatedUser.getId()); + List userNotifications = userNotificationSvc.findByUser(authenticatedUser.getId(), onlyUnread); return ok(Json.createObjectBuilder().add("notifications", json(userNotifications, authenticatedUser, inAppNotificationFormat))); } catch (WrappedResponse wr) { return wr.getResponse(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/NotificationsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/NotificationsIT.java index 2ccb1365d71..c34ab70bef6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/NotificationsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/NotificationsIT.java @@ -87,6 +87,15 @@ public void testNotifications() { Response markRead = UtilIT.markNotificationAsRead(createAccountId, authorApiToken); markRead.then().assertThat().statusCode(OK.getStatusCode()); + // Retrieve only unread notifications + getNotifications = UtilIT.getNotifications(authorApiToken, false, true); + getNotifications.then().assertThat() + .body("data.notifications[0].type", equalTo(CREATEDV.toString())) + .body("data.notifications[0].displayAsRead", equalTo(false)) + .body("data.notifications.size()", equalTo(1)) + .statusCode(OK.getStatusCode()); + + // Retrieve all notifications Response getNotifications2 = UtilIT.getNotifications(authorApiToken); getNotifications2.then().assertThat() .body("data.notifications.size()", equalTo(2)) @@ -184,7 +193,7 @@ public void testNotifications() { .statusCode(OK.getStatusCode()); authorApiToken = UtilIT.getApiTokenFromResponse(createAuthor); - getNotifications = UtilIT.getNotifications(authorApiToken, true); + getNotifications = UtilIT.getNotifications(authorApiToken, true, false); getNotifications.then().assertThat() .body("data.notifications[0].displayAsRead", equalTo(false)) .body("data.notifications.size()", equalTo(1)) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index eba8181e566..4ba8a017f3d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1700,16 +1700,16 @@ static Response returnDatasetToAuthor(String datasetPersistentId, JsonObject jso } static Response getNotifications(String apiToken) { - return getNotifications(apiToken, false); + return getNotifications(apiToken, false, false); } - static Response getNotifications(String apiToken, boolean inAppNotificationFormat) { + static Response getNotifications(String apiToken, boolean inAppNotificationFormat, boolean onlyUnread) { RequestSpecification requestSpecification = given(); if (apiToken != null) { requestSpecification = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken); } - return requestSpecification.get("/api/notifications/all?inAppNotificationFormat=" + inAppNotificationFormat); + return requestSpecification.get("/api/notifications/all?inAppNotificationFormat=" + inAppNotificationFormat + "&onlyUnread=" + onlyUnread); } static Response getUnreadNotificationsCount(String apiToken) { From 4185dc23a5028d8c6f16f13428f1e12d4802e601 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 30 Sep 2025 11:44:14 +0100 Subject: [PATCH 323/634] Added: pagination optional params to getAllNotificationsForUser API endpoint --- .../UserNotificationServiceBean.java | 11 +++- .../iq/dataverse/api/Notifications.java | 6 +- .../iq/dataverse/api/NotificationsIT.java | 63 ++++++++++++++++++- .../edu/harvard/iq/dataverse/api/UtilIT.java | 24 +++++-- 4 files changed, 92 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java index 167b66fb971..b0cc4120800 100644 --- a/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java @@ -43,10 +43,10 @@ public class UserNotificationServiceBean { SettingsServiceBean settingsService; public List findByUser(Long userId) { - return findByUser(userId, false); + return findByUser(userId, false, null, null); } - public List findByUser(Long userId, boolean onlyUnread) { + public List findByUser(Long userId, boolean onlyUnread, Integer limit, Integer offset) { TypedQuery query = em.createQuery( "select un from UserNotification un " + "where un.user.id = :userId and (:onlyUnread = false or un.readNotification = false) " + @@ -57,6 +57,13 @@ public List findByUser(Long userId, boolean onlyUnread) { query.setParameter("userId", userId); query.setParameter("onlyUnread", onlyUnread); + if (offset != null) { + query.setFirstResult(offset); + } + if (limit != null) { + query.setMaxResults(limit); + } + return query.getResultList(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Notifications.java b/src/main/java/edu/harvard/iq/dataverse/api/Notifications.java index c14c2edc19d..a2f692e0710 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Notifications.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Notifications.java @@ -29,10 +29,12 @@ public class Notifications extends AbstractApiBean { @Path("/all") public Response getAllNotificationsForUser(@Context ContainerRequestContext crc, @QueryParam("onlyUnread") boolean onlyUnread, - @QueryParam("inAppNotificationFormat") boolean inAppNotificationFormat) { + @QueryParam("inAppNotificationFormat") boolean inAppNotificationFormat, + @QueryParam("limit") Integer limit, + @QueryParam("offset") Integer offset) { try { AuthenticatedUser authenticatedUser = getRequestAuthenticatedUserOrDie(crc); - List userNotifications = userNotificationSvc.findByUser(authenticatedUser.getId(), onlyUnread); + List userNotifications = userNotificationSvc.findByUser(authenticatedUser.getId(), onlyUnread, limit, offset); return ok(Json.createObjectBuilder().add("notifications", json(userNotifications, authenticatedUser, inAppNotificationFormat))); } catch (WrappedResponse wr) { return wr.getResponse(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/NotificationsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/NotificationsIT.java index c34ab70bef6..1b8c8b61fdb 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/NotificationsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/NotificationsIT.java @@ -88,7 +88,7 @@ public void testNotifications() { markRead.then().assertThat().statusCode(OK.getStatusCode()); // Retrieve only unread notifications - getNotifications = UtilIT.getNotifications(authorApiToken, false, true); + getNotifications = UtilIT.getNotifications(authorApiToken, false, true, null, null); getNotifications.then().assertThat() .body("data.notifications[0].type", equalTo(CREATEDV.toString())) .body("data.notifications[0].displayAsRead", equalTo(false)) @@ -193,7 +193,7 @@ public void testNotifications() { .statusCode(OK.getStatusCode()); authorApiToken = UtilIT.getApiTokenFromResponse(createAuthor); - getNotifications = UtilIT.getNotifications(authorApiToken, true, false); + getNotifications = UtilIT.getNotifications(authorApiToken, true, false, null, null); getNotifications.then().assertThat() .body("data.notifications[0].displayAsRead", equalTo(false)) .body("data.notifications.size()", equalTo(1)) @@ -206,6 +206,65 @@ public void testNotifications() { .body("data.notifications[0].subjectText", equalTo(null)) .body("data.notifications[0].messageText", equalTo(null)) .statusCode(OK.getStatusCode()); + + // Test pagination + + Response authorForPagination = UtilIT.createRandomUser(); + authorForPagination.then().assertThat().statusCode(OK.getStatusCode()); + String paginationApiToken = UtilIT.getApiTokenFromResponse(authorForPagination); + + // Create 4 more notifications to have a total of 5 (1 from CREATEACC, 4 from CREATEDV) + for (int i = 0; i < 4; i++) { + Response dvResponse = UtilIT.createRandomDataverse(paginationApiToken); + dvResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + } + + // Case 1: Test limit + // Get first 2 notifications out of 5 + Response limitedNotifications = UtilIT.getNotifications(paginationApiToken, false, false, 2, 0); + limitedNotifications.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.notifications[0].type", equalTo(CREATEDV.toString())) + .body("data.notifications[1].type", equalTo(CREATEDV.toString())) + .body("data.notifications.size()", equalTo(2)); + + // Case 2: Test offset + // Skip first 3 notifications and get the remaining 2 + Response offsetNotifications = UtilIT.getNotifications(paginationApiToken, false, false, null, 3); + offsetNotifications.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.notifications[0].type", equalTo(CREATEDV.toString())) + .body("data.notifications[1].type", equalTo(CREATEACC.toString())) + .body("data.notifications.size()", equalTo(2)); + + // Case 3: Test limit and offset together + // Get 2 notifications, starting from the 2nd one (index 1) + Response limitedAndOffsetNotifications = UtilIT.getNotifications(paginationApiToken, false, false, 2, 1); + limitedAndOffsetNotifications.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.notifications[0].type", equalTo(CREATEDV.toString())) + .body("data.notifications[1].type", equalTo(CREATEDV.toString())) + .body("data.notifications.size()", equalTo(2)); + + long firstId = JsonPath.from(limitedAndOffsetNotifications.body().asString()).getLong("data.notifications[0].id"); + long secondId = JsonPath.from(limitedAndOffsetNotifications.body().asString()).getLong("data.notifications[1].id"); + + // Verify we got the correct slice of data + Response allNotifications = UtilIT.getNotifications(paginationApiToken); + allNotifications.then().assertThat().statusCode(OK.getStatusCode()); + + long secondNotificationInAll = JsonPath.from(allNotifications.body().asString()).getLong("data.notifications[1].id"); + long thirdNotificationInAll = JsonPath.from(allNotifications.body().asString()).getLong("data.notifications[2].id"); + + assertEquals(secondNotificationInAll, firstId); + assertEquals(thirdNotificationInAll, secondId); + + // Case 4: Test limit larger than available notifications + // Ask for 10, but should only get the 5 that exist + Response excessiveLimitNotifications = UtilIT.getNotifications(paginationApiToken, false, false, 10, 0); + excessiveLimitNotifications.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.notifications.size()", equalTo(5)); } private static void disableSendNotificationOnDatasetCreationSetting() { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 4ba8a017f3d..d8b96e496ba 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1700,16 +1700,28 @@ static Response returnDatasetToAuthor(String datasetPersistentId, JsonObject jso } static Response getNotifications(String apiToken) { - return getNotifications(apiToken, false, false); + return getNotifications(apiToken, false, false, null, null); } - static Response getNotifications(String apiToken, boolean inAppNotificationFormat, boolean onlyUnread) { - RequestSpecification requestSpecification = given(); + static Response getNotifications(String apiToken, boolean inAppNotificationFormat, boolean onlyUnread, Integer limit, Integer offset) { + RequestSpecification request = given(); if (apiToken != null) { - requestSpecification = given() - .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken); + request.header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken); + } + + // Add the standard query parameters + request.queryParam("inAppNotificationFormat", inAppNotificationFormat) + .queryParam("onlyUnread", onlyUnread); + + // Conditionally add pagination parameters only if they are not null + if (limit != null) { + request.queryParam("limit", limit); } - return requestSpecification.get("/api/notifications/all?inAppNotificationFormat=" + inAppNotificationFormat + "&onlyUnread=" + onlyUnread); + if (offset != null) { + request.queryParam("offset", offset); + } + + return request.get("/api/notifications/all"); } static Response getUnreadNotificationsCount(String apiToken) { From 17ade92cd4e036daf7911829fde9d9768df97c10 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 30 Sep 2025 11:56:12 +0100 Subject: [PATCH 324/634] Added: UserNotificationServiceBeanTest --- .../UserNotificationServiceBeanTest.java | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 src/test/java/edu/harvard/iq/dataverse/UserNotificationServiceBeanTest.java diff --git a/src/test/java/edu/harvard/iq/dataverse/UserNotificationServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/UserNotificationServiceBeanTest.java new file mode 100644 index 00000000000..15d3edb95c4 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/UserNotificationServiceBeanTest.java @@ -0,0 +1,123 @@ +package edu.harvard.iq.dataverse; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import jakarta.persistence.EntityManager; +import jakarta.persistence.TypedQuery; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class UserNotificationServiceBeanTest { + + @Mock + private EntityManager em; + + @Mock + private TypedQuery query; + + @InjectMocks + private UserNotificationServiceBean userNotificationService; + + @BeforeEach + void setUp() { + when(em.createQuery(anyString(), eq(UserNotification.class))).thenReturn(query); + } + + @Test + void testFindByUser_withoutPagination() { + // Arrange + Long userId = 1L; + List expectedNotifications = Arrays.asList(new UserNotification(), new UserNotification()); + when(query.getResultList()).thenReturn(expectedNotifications); + + // Act + List actualNotifications = userNotificationService.findByUser(userId, false, null, null); + + // Assert + assertEquals(2, actualNotifications.size()); + verify(query).setParameter("userId", userId); + verify(query).setParameter("onlyUnread", false); + // Verify that pagination methods were never called + verify(query, never()).setFirstResult(anyInt()); + verify(query, never()).setMaxResults(anyInt()); + } + + @Test + void testFindByUser_withOnlyUnread() { + // Arrange + Long userId = 1L; + when(query.getResultList()).thenReturn(Collections.emptyList()); + + // Act + userNotificationService.findByUser(userId, true, null, null); + + // Assert + verify(query).setParameter("userId", userId); + verify(query).setParameter("onlyUnread", true); + } + + @Test + void testFindByUser_withPagination() { + // Arrange + Long userId = 1L; + int limit = 10; + int offset = 20; + when(query.getResultList()).thenReturn(Collections.emptyList()); + + // Act + userNotificationService.findByUser(userId, false, limit, offset); + + // Assert + verify(query).setParameter("userId", userId); + verify(query).setParameter("onlyUnread", false); + // Verify that pagination methods were called with the correct values + verify(query).setFirstResult(offset); + verify(query).setMaxResults(limit); + } + + @Test + void testFindByUser_withOnlyLimit() { + // Arrange + Long userId = 1L; + int limit = 5; + when(query.getResultList()).thenReturn(Collections.emptyList()); + + // Act + userNotificationService.findByUser(userId, false, limit, null); + + // Assert + verify(query).setMaxResults(limit); + verify(query, never()).setFirstResult(anyInt()); + } + + @Test + void testFindByUser_withOnlyOffset() { + // Arrange + Long userId = 1L; + int offset = 15; + when(query.getResultList()).thenReturn(Collections.emptyList()); + + // Act + userNotificationService.findByUser(userId, false, null, offset); + + // Assert + verify(query).setFirstResult(offset); + verify(query, never()).setMaxResults(anyInt()); + } +} From 710822cf77599f4bed666cb24d04bf28326879c4 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 30 Sep 2025 12:01:17 +0100 Subject: [PATCH 325/634] Changed: docs for Get All Notifications by User --- doc/sphinx-guides/source/api/native-api.rst | 24 +++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index fa4b4611559..03166c87f4f 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -6129,13 +6129,33 @@ The expected OK (200) response looks something like this: } ... -This endpoint supports an optional query parameter ``inAppNotificationFormat`` which, if sent as ``true``, retrieves the fields needed to build the in-app notifications for the Notifications section of the Dataverse UI, omitting fields related to email notifications. +This endpoint supports several optional query parameters to filter and paginate the results. + +The ``inAppNotificationFormat`` parameter, if sent as ``true``, retrieves the fields needed to build the in-app notifications for the Notifications section of the Dataverse UI, omitting fields related to email notifications. .. code-block:: bash curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/notifications/all?inAppNotificationFormat=true" -The expected OK (200) response looks something like this: +The ``onlyUnread`` parameter, if sent as ``true``, filters the results to include only notifications that have not been marked as read. + +.. code-block:: bash + + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/notifications/all?onlyUnread=true" + +The ``limit`` and ``offset`` parameters can be used for pagination. ``limit`` specifies the maximum number of notifications to return, and ``offset`` specifies the number of notifications to skip from the beginning of the list. For example, to retrieve notifications 11 through 15: + +.. code-block:: bash + + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/notifications/all?limit=5&offset=10" + +All parameters can be combined. For instance, to get the first page of 10 unread notifications in the in-app format: + +.. code-block:: bash + + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/notifications/all?inAppNotificationFormat=true&onlyUnread=true&limit=10&offset=0" + +The expected OK (200) response for an in-app format request looks something like this: .. code-block:: text From e68eda0f7c7b7ff8a82f7dfd579d573b4374c901 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 30 Sep 2025 12:09:29 +0100 Subject: [PATCH 326/634] Added: release notes for #11852 --- ...ications-api-pagination-and-only-unread.md | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 doc/release-notes/11852-notifications-api-pagination-and-only-unread.md diff --git a/doc/release-notes/11852-notifications-api-pagination-and-only-unread.md b/doc/release-notes/11852-notifications-api-pagination-and-only-unread.md new file mode 100644 index 00000000000..014e22c7739 --- /dev/null +++ b/doc/release-notes/11852-notifications-api-pagination-and-only-unread.md @@ -0,0 +1,37 @@ +## Notifications API Update + +**Endpoint:** `notifications/all` + +The user notifications endpoint has been enhanced with new optional query parameters to allow for more specific and +efficient data retrieval. + +**1. Filter by Unread Status** + +You can now fetch only unread notifications by using the `onlyUnread` boolean parameter. + +* **`onlyUnread`**: (Optional, boolean) When set to `true`, the API will only return notifications that the user has not + yet marked as read. + +**Example:** + +```bash +curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/notifications/all?onlyUnread=true" +``` + +**2. Pagination Support** + +Pagination is now supported through the limit and offset parameters, allowing you to retrieve notifications in smaller, +manageable chunks. + +- **`limit`**: (Optional, integer) Specifies the maximum number of notifications to return. + +- **`offset`**: (Optional, integer) Specifies the number of notifications to skip before starting to return results. + +Example (Retrieve notifications 11 through 20): + +```bash +curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/notifications/all?limit=10&offset=10" +``` + +Related issue: #11852 +Related PR: #11854 \ No newline at end of file From ef1803b6701add59f6d5d3b12d408d10cab91442 Mon Sep 17 00:00:00 2001 From: Akos Kukucska Date: Tue, 30 Sep 2025 16:54:08 +0200 Subject: [PATCH 327/634] Create a new API endpoint for querying metadata language. --- .../harvard/iq/dataverse/api/Dataverses.java | 17 +++++++++++++++++ .../iq/dataverse/util/json/JsonPrinter.java | 10 ++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index ae82ff46522..79fb95daff4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1969,4 +1969,21 @@ public Response createTemplate(@Context ContainerRequestContext crc, String body return e.getResponse(); } } + + @GET + @AuthRequired + @Path("{identifier}/metadataLanguage") + public Response getMetadataLanguage(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf) { + try { + Map langMap = settingsService.getBaseMetadataLanguageMap(null, true); + Dataverse dataverse = findDataverseOrDie(dvIdtf); + String dvMetadataLanguage = dataverse.getMetadataLanguage(); + if (!dvMetadataLanguage.equals(DvObjectContainer.UNDEFINED_CODE)) { + return ok(Json.createArrayBuilder().add(jsonLanguage(dvMetadataLanguage, langMap.get(dvMetadataLanguage)))); + } + return ok(jsonLanguage(langMap)); + } catch (WrappedResponse e) { + return e.getResponse(); + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index bbc834e0cc4..577cc08dc57 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -1685,4 +1685,14 @@ public static JsonArrayBuilder json(List notifications, Authen return notificationsArray; } + + public static JsonObjectBuilder jsonLanguage(String locale, String title) { + // returns a single metadata language entry + return jsonObjectBuilder().add("locale", locale).add("title", title); + } + + public static JsonArrayBuilder jsonLanguage(Map langMap) { + // returns an array of metadatalanguages + return Json.createArrayBuilder(langMap.entrySet().stream().map(entry -> jsonLanguage(entry.getKey(), entry.getValue())).toList()); + } } From 3c033623422ba263503fc78e6bac6ea65fbf4e12 Mon Sep 17 00:00:00 2001 From: Akos Kukucska Date: Tue, 30 Sep 2025 18:06:31 +0200 Subject: [PATCH 328/634] Move logic to a command for authentication --- .../harvard/iq/dataverse/api/Dataverses.java | 14 ++----- .../GetDataverseMetadataLanguageCommand.java | 41 +++++++++++++++++++ 2 files changed, 45 insertions(+), 10 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseMetadataLanguageCommand.java diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 79fb95daff4..46cdf9d670d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1974,16 +1974,10 @@ public Response createTemplate(@Context ContainerRequestContext crc, String body @AuthRequired @Path("{identifier}/metadataLanguage") public Response getMetadataLanguage(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf) { - try { - Map langMap = settingsService.getBaseMetadataLanguageMap(null, true); + return response(req -> { Dataverse dataverse = findDataverseOrDie(dvIdtf); - String dvMetadataLanguage = dataverse.getMetadataLanguage(); - if (!dvMetadataLanguage.equals(DvObjectContainer.UNDEFINED_CODE)) { - return ok(Json.createArrayBuilder().add(jsonLanguage(dvMetadataLanguage, langMap.get(dvMetadataLanguage)))); - } - return ok(jsonLanguage(langMap)); - } catch (WrappedResponse e) { - return e.getResponse(); - } + return ok(jsonLanguage(execCommand( + new GetDataverseMetadataLanguageCommand(req, dataverse)))); + }, getRequestUser(crc)); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseMetadataLanguageCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseMetadataLanguageCommand.java new file mode 100644 index 00000000000..82b5527048b --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseMetadataLanguageCommand.java @@ -0,0 +1,41 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DvObjectContainer; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +public class GetDataverseMetadataLanguageCommand extends AbstractCommand> { + + private final Dataverse dv; + + public GetDataverseMetadataLanguageCommand(DataverseRequest aRequest, Dataverse dv) { + super(aRequest, dv); + this.dv = dv; + } + + @Override + public Map execute(CommandContext ctxt) throws CommandException { + Map langMap = ctxt.settings().getBaseMetadataLanguageMap(null, true); + String dvMetadataLanguage = dv.getMetadataLanguage(); + if (!dvMetadataLanguage.equals(DvObjectContainer.UNDEFINED_CODE)) { + return Collections.singletonMap(dvMetadataLanguage, langMap.get(dvMetadataLanguage)); + } + return langMap; + + } + + @Override + public Map> getRequiredPermissions() { + return Collections.singletonMap("", + dv.isReleased() ? Collections.emptySet() + : Collections.singleton(Permission.ViewUnpublishedDataverse)); + } +} From 94eca41d70283c09591443e3912cc530cf7f7add Mon Sep 17 00:00:00 2001 From: Omer Fahim Date: Tue, 30 Sep 2025 14:03:57 -0400 Subject: [PATCH 329/634] Update doc/sphinx-guides/source/container/app-image.rst typos fix --- doc/sphinx-guides/source/container/app-image.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/container/app-image.rst b/doc/sphinx-guides/source/container/app-image.rst index 12f5984a66b..47055bd3786 100644 --- a/doc/sphinx-guides/source/container/app-image.rst +++ b/doc/sphinx-guides/source/container/app-image.rst @@ -81,7 +81,7 @@ For now, stale images will be kept on Docker Hub indefinitely. | Example: :substitution-code:`|nextVersion|-noble` | Summary: Rolling tag, equivalent to ``unstable`` for current development cycle. Will roll over to the rolling production tag after a Dataverse release. - | Discussion: Perhaps you are eager to starting testing features of an upcoming version (e.g. |nextVersion|) in a staging environment. You select the :substitution-code:`|nextVersion|-noble` tag (as opposed to ``unstable``) because you want to stay on |nextVersion| rather than switching to the version **after that** when a release is made (which would happen if you had selected the ``unstable`` tag). Also, when the next release comes out (|nextVersion| in this example), you would stay on the :substitution-code:`|nextVersion|-noble` tag, which is the same tag that someone would use who wants the final release of |nextVersion|. (See "Rolling Production", above.) + | Discussion: Perhaps you are eager to start testing features of an upcoming version (e.g. |nextVersion|) in a staging environment. You select the :substitution-code:`|nextVersion|-noble` tag (as opposed to ``unstable``) because you want to stay on |nextVersion| rather than switching to the version **after that** when a release is made (which would happen if you had selected the ``unstable`` tag). Also, when the next release comes out (|nextVersion| in this example), you would stay on the :substitution-code:`|nextVersion|-noble` tag, which is the same tag that someone would use who wants the final release of |nextVersion|. (See "Rolling Production", above.) **NOTE**: In these tags for development usage, the version number will always be 1 minor version ahead of existing Dataverse releases. Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. From 9933f1678b8b4a1410b7a409f0ae65d1ab3481cf Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 30 Sep 2025 16:00:13 -0400 Subject: [PATCH 330/634] #11772 add content test --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index bb4793961f8..9fe30cd451e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -4092,7 +4092,8 @@ public void testUpdateDatasetTerms() throws IOException { updateTerms = UtilIT.updateDatasetTermsAndAccess(datasetPersistentId, apiToken, pathToJsonFile); updateTerms.prettyPrint(); updateTerms.then().assertThat() - .statusCode(OK.getStatusCode()); + .statusCode(OK.getStatusCode()) + .body("data.termsOfAccess", equalTo("For access to restriced files please see read me file")); // Restrict file Response restrictFileResponse = UtilIT.restrictFile(fileId, true, apiToken); From 3649cd4d8ff43ad2e300bcb17f83da5d9c32b337 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 30 Sep 2025 16:00:47 -0400 Subject: [PATCH 331/634] #11772 fix typo --- .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index bbc834e0cc4..0b11a5d1fd8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -495,7 +495,7 @@ public static JsonObjectBuilder json(FileDetailsHolder ds) { } public static JsonObjectBuilder json(DatasetVersion dsv, boolean includeFiles) { - return json(dsv, null, includeFiles, false,true); + return json(dsv, null, includeFiles, false, true); } public static JsonObjectBuilder json(DatasetVersion dsv, boolean includeFiles, boolean includeMetadataBlocks) { return json(dsv, null, includeFiles, false, includeMetadataBlocks); From d901f651556272a8e7a441291c3d23668d2a4c7c Mon Sep 17 00:00:00 2001 From: Akos Kukucska Date: Wed, 1 Oct 2025 11:32:42 +0200 Subject: [PATCH 332/634] Test the dataverse metadataLanguage API endpoint --- .../iq/dataverse/api/DataversesIT.java | 31 +++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 8 +++++ 2 files changed, 39 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 155166a386d..cc764e48286 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -2457,4 +2457,35 @@ private String getSuperuserToken() { UtilIT.makeSuperUser(username); return adminApiToken; } + + @Test + public void testGetDataverseMetadataLanguage() { + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + UtilIT.deleteSetting(":MetadataLanguages"); + Response createDataverse1Response = UtilIT.createRandomDataverse(apiToken); + + createDataverse1Response.prettyPrint(); + createDataverse1Response.then().assertThat().statusCode(CREATED.getStatusCode()); + + String alias = UtilIT.getAliasFromResponse(createDataverse1Response); + + Response noLang = UtilIT.getDataverseMetadataLanguage(alias, apiToken); + noLang.prettyPrint(); + + noLang.then().assertThat().body("data", equalTo(List.of())); + + UtilIT.setSetting(":MetadataLanguages", + "[{\"locale\":\"en\",\"title\":\"English\"},{\"locale\":\"hu\",\"title\":\"magyar\"}]"); + Response allLangs = UtilIT.getDataverseMetadataLanguage(alias, apiToken); + allLangs.prettyPrint(); + allLangs.then().assertThat() + .body("data.size()", equalTo(2)) + .and().body("data[0].locale", equalTo("en")) + .and().body("data[1].locale", equalTo("hu")); + + } + } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index e5d3f908349..c5c38aa7e98 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -5001,4 +5001,12 @@ public static Response callCallbackUrl(String callbackUrl) { .when() .get(callbackUrl); } + + public static Response getDataverseMetadataLanguage(String alias, String apiToken) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/dataverses/" + + alias + + "/metadataLanguage"); + } } From 2304ac4392edee977df10417632873b349adbcea Mon Sep 17 00:00:00 2001 From: Akos Kukucska Date: Wed, 1 Oct 2025 16:48:40 +0200 Subject: [PATCH 333/634] Add release note --- doc/release-notes/metadataLanguage-API-call.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/release-notes/metadataLanguage-API-call.md diff --git a/doc/release-notes/metadataLanguage-API-call.md b/doc/release-notes/metadataLanguage-API-call.md new file mode 100644 index 00000000000..a0b68f7ad13 --- /dev/null +++ b/doc/release-notes/metadataLanguage-API-call.md @@ -0,0 +1,5 @@ +A new API endpoint has been implemented for getting the metadata language of a Dataverse Collection: + +`GET /dataverses/{alias}/metadataLanguage`: Returns the specified metadata language(s) in the collection if any. + +For more information, see #11856 and #11856. \ No newline at end of file From 50a321f96682193bf554a8c14266cfb5cf773a1b Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 1 Oct 2025 17:15:06 +0100 Subject: [PATCH 334/634] Stash: refactoring getCompareVersionsSummary endpoint WIP --- .../harvard/iq/dataverse/api/Datasets.java | 13 +++- .../DatasetVersionSummary.java | 72 +++++++++++++++++++ .../DatasetVersionSummaryContent.java | 4 ++ ...setVersionSummaryContentDeaccessioned.java | 20 ++++++ ...tasetVersionSummaryContentDifferences.java | 4 ++ .../DatasetVersionSummaryContentSimple.java | 16 +++++ .../GetDatasetVersionSummariesCommand.java | 39 ++++++++++ .../iq/dataverse/util/json/JsonPrinter.java | 6 ++ 8 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummary.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContent.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentDeaccessioned.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentDifferences.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentSimple.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommand.java diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 3334f2ec8bb..b2a93f7b2b0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3138,8 +3138,15 @@ public Response getCompareVersions(@Context ContainerRequestContext crc, @PathPa @GET @AuthRequired @Path("{id}/versions/compareSummary") - public Response getCompareVersionsSummary(@Context ContainerRequestContext crc, @PathParam("id") String id, - @Context UriInfo uriInfo, @Context HttpHeaders headers) { + public Response getCompareVersionsSummary(@Context ContainerRequestContext crc, @PathParam("id") String id) { + return response(req -> { + try { + return ok(json(execCommand(new GetDatasetVersionSummariesCommand(req, findDatasetOrDie(id))))); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + }, getRequestUser(crc)); + /* try { Dataset dataset = findDatasetOrDie(id); User user = getRequestUser(crc); @@ -3188,7 +3195,7 @@ public Response getCompareVersionsSummary(@Context ContainerRequestContext crc, return ok(differenceSummaries); } catch (WrappedResponse wr) { return wr.getResponse(); - } + }*/ } private JsonObject getDeaccessionJson(DatasetVersion dv) { diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummary.java b/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummary.java new file mode 100644 index 00000000000..4a34e406398 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummary.java @@ -0,0 +1,72 @@ +package edu.harvard.iq.dataverse.datasetversionsummaries; + +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.DatasetVersion.VersionState; +import edu.harvard.iq.dataverse.datasetversionsummaries.DatasetVersionSummaryContentSimple.Content; + +import java.util.Optional; + +/** + * An immutable data carrier representing a summary of a DatasetVersion. + */ +public record DatasetVersionSummary( + String versionId, + String versionNote, + String contributorNames, + String publicationDate, + DatasetVersionSummaryContent content +) { + + /** + * Creates a DatasetVersionSummary from a DatasetVersion entity. + *

      + * This factory method is the preferred way to create summaries, as it encapsulates + * all the logic for determining the correct summary content. + * + * @param datasetVersion The entity to convert. + * @return An {@link Optional} containing the summary, or an empty Optional if the input is null. + */ + public static Optional from(DatasetVersion datasetVersion) { + return Optional.ofNullable(datasetVersion).map(version -> new DatasetVersionSummary( + version.getFriendlyVersionNumber(), + version.getVersionNote(), + version.getContributorNames(), + version.getPublicationDateAsString(), + determineContent(version) + )); + } + + /** + * Determines the appropriate summary content based on the version's state. + * The logic is prioritized to handle the most specific cases first. + * + * @return The determined {@link DatasetVersionSummaryContent}, or {@code null} if the version + * state does not match any known summary condition. + */ + private static DatasetVersionSummaryContent determineContent(DatasetVersion version) { + // Priority 1: The version has explicit differences calculated. + if (version.getDefaultVersionDifference() != null) { + // TODO + return new DatasetVersionSummaryContentDifferences(); + } + + // Priority 2: Deaccessioned status is a critical override. + if (version.isDeaccessioned()) { + return new DatasetVersionSummaryContentDeaccessioned(version.getDeaccessionNote(), version.getDeaccessionLink()); + } + + // Priority 3: Standard draft or released versions. + if (version.isReleased() || version.isDraft()) { + boolean isFirstVersion = version.getPriorVersionState() == null; + if (isFirstVersion) { + return new DatasetVersionSummaryContentSimple(version.isReleased() ? Content.firstPublished : Content.firstDraft); + } + + if (version.getPriorVersionState() == VersionState.DEACCESSIONED) { + return new DatasetVersionSummaryContentSimple(Content.previousVersionDeaccessioned); + } + } + + return null; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContent.java b/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContent.java new file mode 100644 index 00000000000..653c13aa970 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContent.java @@ -0,0 +1,4 @@ +package edu.harvard.iq.dataverse.datasetversionsummaries; + +public abstract class DatasetVersionSummaryContent { +} diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentDeaccessioned.java b/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentDeaccessioned.java new file mode 100644 index 00000000000..6a6ea3fc496 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentDeaccessioned.java @@ -0,0 +1,20 @@ +package edu.harvard.iq.dataverse.datasetversionsummaries; + +public class DatasetVersionSummaryContentDeaccessioned extends DatasetVersionSummaryContent { + + private final String deaccessionNote; + private final String deaccessionLink; + + public DatasetVersionSummaryContentDeaccessioned(String deaccessionNote, String deaccessionLink) { + this.deaccessionNote = deaccessionNote; + this.deaccessionLink = deaccessionLink; + } + + public String getDeaccessionNote() { + return deaccessionNote; + } + + public String getDeaccessionLink() { + return deaccessionLink; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentDifferences.java b/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentDifferences.java new file mode 100644 index 00000000000..c20eebfc400 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentDifferences.java @@ -0,0 +1,4 @@ +package edu.harvard.iq.dataverse.datasetversionsummaries; + +public class DatasetVersionSummaryContentDifferences extends DatasetVersionSummaryContent { +} diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentSimple.java b/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentSimple.java new file mode 100644 index 00000000000..db0172349f6 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentSimple.java @@ -0,0 +1,16 @@ +package edu.harvard.iq.dataverse.datasetversionsummaries; + +public class DatasetVersionSummaryContentSimple extends DatasetVersionSummaryContent { + + private Content value; + + public DatasetVersionSummaryContentSimple(Content value) { + this.value = value; + } + + public enum Content { + firstPublished, + previousVersionDeaccessioned, + firstDraft + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommand.java new file mode 100644 index 00000000000..23ea0658f0a --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommand.java @@ -0,0 +1,39 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.datasetversionsummaries.DatasetVersionSummary; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +import java.util.EnumSet; +import java.util.List; +import java.util.Optional; + +public class GetDatasetVersionSummariesCommand extends AbstractCommand> { + + private final Dataset dataset; + + public GetDatasetVersionSummariesCommand(DataverseRequest request, Dataset dataset) { + super(request, dataset); + this.dataset = dataset; + } + + @Override + public List execute(CommandContext ctxt) throws CommandException { + return dataset.getVersions().stream() + .filter(dv -> versionRequiresSummary(ctxt, dv)) + .map(DatasetVersionSummary::from) + .flatMap(Optional::stream) + .toList(); + } + + private boolean versionRequiresSummary(CommandContext ctxt, DatasetVersion dv) { + return dv.isPublished() + || dv.isDeaccessioned() + || ctxt.permissions().hasPermissionsFor(getRequest().getUser(), dv.getDataset(), EnumSet.of(Permission.ViewUnpublishedDataset)); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index bbc834e0cc4..c6d3a18be87 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -19,6 +19,7 @@ import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataset.DatasetType; import edu.harvard.iq.dataverse.dataset.DatasetUtil; +import edu.harvard.iq.dataverse.datasetversionsummaries.DatasetVersionSummary; import edu.harvard.iq.dataverse.datavariable.CategoryMetadata; import edu.harvard.iq.dataverse.datavariable.DataVariable; import edu.harvard.iq.dataverse.datavariable.SummaryStatistic; @@ -1685,4 +1686,9 @@ public static JsonArrayBuilder json(List notifications, Authen return notificationsArray; } + + public static JsonArrayBuilder json(List datasetVersionSummaries) { + // TODO + return Json.createArrayBuilder(); + } } From e98a4f5af37b8dfeb59d65c8f9362202643fd22e Mon Sep 17 00:00:00 2001 From: "@suraj05" Date: Wed, 1 Oct 2025 22:38:45 +0530 Subject: [PATCH 335/634] Fix typo in demo.rst: change "collection" to "collections" in User Guide reference --- doc/sphinx-guides/source/container/running/demo.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/container/running/demo.rst b/doc/sphinx-guides/source/container/running/demo.rst index d4afee8a18a..32de0ea48bf 100644 --- a/doc/sphinx-guides/source/container/running/demo.rst +++ b/doc/sphinx-guides/source/container/running/demo.rst @@ -261,7 +261,7 @@ You should be able to see the new fields from the metadata block you added in th ``curl http://localhost:8983/solr/collection1/schema/fields`` -At this point you can proceed with testing the metadata block in the Dataverse UI. First you'll need to enable it for a collection (see :ref:`general-information` in the User Guide section about collection). Afterwards, create a new dataset, save it, and then edit the metadata for that dataset. Your metadata block should appear. +At this point you can proceed with testing the metadata block in the Dataverse UI. First you'll need to enable it for a collection (see :ref:`general-information` in the User Guide section about collections). Afterwards, create a new dataset, save it, and then edit the metadata for that dataset. Your metadata block should appear. Next Steps ---------- From c0c1f86aa1b806851820b756bc017e128e8f80d7 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 1 Oct 2025 16:04:55 -0400 Subject: [PATCH 336/634] #11772 public install update/test --- .../harvard/iq/dataverse/api/Datasets.java | 9 +++++- .../harvard/iq/dataverse/api/DatasetsIT.java | 29 ++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 823b5674c32..72ed1158788 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1159,6 +1159,9 @@ public Response editVersionMetadata(@Context ContainerRequestContext crc, String public Response editVersionTermsOfUse(@Context ContainerRequestContext crc, String jsonBody, @PathParam("id") String id, @QueryParam("sourceLastUpdateTime") String sourceLastUpdateTime) { try { + + boolean publicInstall = settingsSvc.isTrueForKey(SettingsServiceBean.Key.PublicInstall, false); + Dataset dataset = findDatasetOrDie(id); if (sourceLastUpdateTime != null) { @@ -1166,9 +1169,13 @@ public Response editVersionTermsOfUse(@Context ContainerRequestContext crc, Stri } JsonObject json = JsonUtil.getJsonObject(jsonBody); - System.out.print(jsonBody); + TermsOfUseAndAccess toua = jsonParser().parseTermsOfUseAndAccess(json); + if (publicInstall && (toua.isFileAccessRequest() || !toua.getTermsOfAccess().isEmpty())){ + return error(BAD_REQUEST, "Setting File Access Request or Terms of Access is not permitted on a public installation."); + } + toua.setDatasetVersion(dataset.getLatestVersion()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index abd37d1b830..1a128c14bee 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -4094,6 +4094,15 @@ public void testFilesUnchangedAfterDatasetMetadataUpdate() throws IOException { @Test public void testUpdateDatasetTerms() throws IOException { + + //get publicInstall setting so we can change it back + Response publicInstallResponse = UtilIT.getSetting(SettingsServiceBean.Key.PublicInstall); + //TODO: fix this its a little hacky + String publicInstall = String.valueOf(publicInstallResponse.getBody().asString().contains("true")); + + // make sure this is not a public installation + UtilIT.setSetting(SettingsServiceBean.Key.PublicInstall, "false"); + Response createUser = UtilIT.createRandomUser(); createUser.prettyPrint(); createUser.then().assertThat() @@ -4163,7 +4172,25 @@ public void testUpdateDatasetTerms() throws IOException { Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetPersistentId, "major", apiToken); publishDatasetResponse.prettyPrint(); publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode()); - + + //Make installation "public install" to see that terms of access may not be set + UtilIT.setSetting(SettingsServiceBean.Key.PublicInstall, "true"); + + createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + updateTerms = UtilIT.updateDatasetTermsAndAccess(datasetPid, apiToken, pathToJsonFile); + updateTerms.prettyPrint(); + // should fail because its now a public install + updateTerms.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo("Setting File Access Request or Terms of Access is not permitted on a public installation.")); + //Setting File Access Request or Terms of Access is not permitted on a public installation. + + //reset public install + UtilIT.setSetting(SettingsServiceBean.Key.PublicInstall, publicInstall); } From dfe334c6ffafc7a243e249ddf216e8460cb86d2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Oct 2025 23:14:36 +0000 Subject: [PATCH 337/634] Bump peter-evans/dockerhub-description from 4 to 5 Bumps [peter-evans/dockerhub-description](https://github.com/peter-evans/dockerhub-description) from 4 to 5. - [Release notes](https://github.com/peter-evans/dockerhub-description/releases) - [Commits](https://github.com/peter-evans/dockerhub-description/compare/v4...v5) --- updated-dependencies: - dependency-name: peter-evans/dockerhub-description dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/container_maintenance.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 142363cbe1a..d863f838881 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -218,7 +218,7 @@ jobs: cat "./modules/container-base/README.md" - name: Push description to DockerHub for base image if: ${{ ! inputs.dry_run && ! inputs.damp_run && toJSON(needs.base-image.outputs.rebuilt_images) != '[]' }} - uses: peter-evans/dockerhub-description@v4 + uses: peter-evans/dockerhub-description@v5 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -243,7 +243,7 @@ jobs: cat "./src/main/docker/README.md" - name: Push description to DockerHub for application image if: ${{ ! inputs.dry_run && ! inputs.damp_run && toJSON(needs.application-image.outputs.rebuilt_images) != '[]' }} - uses: peter-evans/dockerhub-description@v4 + uses: peter-evans/dockerhub-description@v5 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -268,7 +268,7 @@ jobs: cat "./modules/container-configbaker/README.md" - name: Push description to DockerHub for config baker image if: ${{ ! inputs.dry_run && ! inputs.damp_run && toJSON(needs.configbaker-image.outputs.rebuilt_images) != '[]' }} - uses: peter-evans/dockerhub-description@v4 + uses: peter-evans/dockerhub-description@v5 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} From c6982988ded89e0b0364b59a863e5829a9bd03a3 Mon Sep 17 00:00:00 2001 From: Akos Kukucska Date: Thu, 2 Oct 2025 11:38:44 +0200 Subject: [PATCH 338/634] Implement an API endpoint for setting a Dataverse collection metadata language. --- .../metadataLanguage-API-call.md | 3 +- doc/sphinx-guides/source/api/native-api.rst | 45 +++++++++++++++++++ .../harvard/iq/dataverse/api/Dataverses.java | 20 ++++++++- .../SetDataverseMetadataLanguageCommand.java | 33 ++++++++++++++ .../iq/dataverse/api/DataversesIT.java | 14 ++++-- .../edu/harvard/iq/dataverse/api/UtilIT.java | 11 ++++- 6 files changed, 120 insertions(+), 6 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetDataverseMetadataLanguageCommand.java diff --git a/doc/release-notes/metadataLanguage-API-call.md b/doc/release-notes/metadataLanguage-API-call.md index a0b68f7ad13..9e2598cf159 100644 --- a/doc/release-notes/metadataLanguage-API-call.md +++ b/doc/release-notes/metadataLanguage-API-call.md @@ -1,5 +1,6 @@ A new API endpoint has been implemented for getting the metadata language of a Dataverse Collection: -`GET /dataverses/{alias}/metadataLanguage`: Returns the specified metadata language(s) in the collection if any. +`GET /dataverses/{alias}/allowedMetadataLanguages`: Returns the specified metadata language(s) in the collection if any. +`PUT /dataverses/{alias}/allowedMetadataLanguages{metadataLanguage}`: Sets a metadata language in the collection. For more information, see #11856 and #11856. \ No newline at end of file diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 19a4d79c5ae..1cef814040e 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -287,6 +287,51 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/roles" +List the Allowed Metadata Languages of a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Shows the allowed metadata languages of the Dataverse collection ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=root + + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/allowedMetadataLanguages" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/allowedMetadataLanguages" + +If there are no metadata languages configured on the server, this call returns an empty array. If the Dataverse collection has a mandatory metadata language, the return value is an array of that single language, +otherwise it's an array of all available metadata languages on the server. + +Set the Allowed Metadata Language of a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Sets the allowed metadata language of the Dataverse collection ``id`` to ``langCode`` if it's available on the server: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=root + export LANGCODE=en + + curl -H "X-Dataverse-key:$API_TOKEN" -X PUT "$SERVER_URL/api/dataverses/$ID/allowedMetadataLanguages/$LANGCODE" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/dataverses/root/allowedMetadataLanguages/en" + +Returns an array of the set metadata language. +If the metadata language is not available on the server, this call responds with a 400 BAD REQUEST. + List Facets Configured for a Dataverse Collection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 46cdf9d670d..a2e087a4a9a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -72,6 +72,7 @@ import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; +import org.springframework.security.access.method.P; import javax.xml.stream.XMLStreamException; @@ -1972,7 +1973,7 @@ public Response createTemplate(@Context ContainerRequestContext crc, String body @GET @AuthRequired - @Path("{identifier}/metadataLanguage") + @Path("{identifier}/allowedMetadataLanguages") public Response getMetadataLanguage(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf) { return response(req -> { Dataverse dataverse = findDataverseOrDie(dvIdtf); @@ -1980,4 +1981,21 @@ public Response getMetadataLanguage(@Context ContainerRequestContext crc, @PathP new GetDataverseMetadataLanguageCommand(req, dataverse)))); }, getRequestUser(crc)); } + + @PUT + @AuthRequired + @Path("{identifier}/allowedMetadataLanguages/{metadataLanguage}") + public Response setMetadataLanguage(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @PathParam("metadataLanguage") String lang) { + return response(req -> { + Map langMap = settingsService.getBaseMetadataLanguageMap(null, true); + if (langMap.isEmpty()) { + return badRequest("There are no metadata languages configured on this server"); + } + if (!langMap.containsKey(lang)) { + return badRequest("The specified metadata language " + lang + " is not allowed on this server!"); + } + Dataverse dataverse = findDataverseOrDie(dvIdtf); + return ok(jsonLanguage(execCommand(new SetDataverseMetadataLanguageCommand(req, dataverse, lang)))); + }, getRequestUser(crc)); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetDataverseMetadataLanguageCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetDataverseMetadataLanguageCommand.java new file mode 100644 index 00000000000..438c332436a --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetDataverseMetadataLanguageCommand.java @@ -0,0 +1,33 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import java.util.Collections; +import java.util.Map; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +@RequiredPermissions(Permission.EditDataverse) +public class SetDataverseMetadataLanguageCommand extends AbstractCommand> { + + private Dataverse dv; + private String lang; + + public SetDataverseMetadataLanguageCommand(DataverseRequest aRequest, Dataverse dv, String lang) { + super(aRequest, dv); + this.dv = dv; + this.lang = lang; + } + + @Override + public Map execute(CommandContext ctxt) throws CommandException { + dv.setMetadataLanguage(lang); + return Collections.singletonMap(lang, ctxt.settings().getBaseMetadataLanguageMap(null, true).get(lang)); + } + +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index cc764e48286..2540ddc2fca 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -20,6 +20,7 @@ import java.text.MessageFormat; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.logging.Logger; import jakarta.json.Json; @@ -30,6 +31,8 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Isolated; +import org.junit.jupiter.api.parallel.ResourceLock; import static jakarta.ws.rs.core.Response.Status.*; import static org.hamcrest.CoreMatchers.*; @@ -2459,10 +2462,10 @@ private String getSuperuserToken() { } @Test - public void testGetDataverseMetadataLanguage() { + @ResourceLock(value = "") + public void testDataverseMetadataLanguage() { Response createUser = UtilIT.createRandomUser(); createUser.prettyPrint(); - String username = UtilIT.getUsernameFromResponse(createUser); String apiToken = UtilIT.getApiTokenFromResponse(createUser); UtilIT.deleteSetting(":MetadataLanguages"); Response createDataverse1Response = UtilIT.createRandomDataverse(apiToken); @@ -2485,7 +2488,12 @@ public void testGetDataverseMetadataLanguage() { .body("data.size()", equalTo(2)) .and().body("data[0].locale", equalTo("en")) .and().body("data[1].locale", equalTo("hu")); - + + Response english = UtilIT.setDataverseMetadataLanguage(alias, apiToken, "en"); + english.then().assertThat().body("data", equalTo(List.of(Map.of("locale", "en", "title", "English")))); + Response singleLang = UtilIT.getDataverseMetadataLanguage(alias, apiToken); + singleLang.then().assertThat().body("data", equalTo(List.of(Map.of("locale", "en", "title", "English")))); + UtilIT.deleteSetting(":MetadataLanguages"); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index c5c38aa7e98..3888381bbc4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -5007,6 +5007,15 @@ public static Response getDataverseMetadataLanguage(String alias, String apiToke .header(API_TOKEN_HTTP_HEADER, apiToken) .get("/api/dataverses/" + alias - + "/metadataLanguage"); + + "/allowedMetadataLanguages"); + } + + public static Response setDataverseMetadataLanguage(String alias, String apiToken, String lang) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .put("/api/dataverses/" + + alias + + "/allowedMetadataLanguages/" + + lang); } } From 7d1652239773e87f55f572cd9306a15428407e1a Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 2 Oct 2025 09:17:39 -0400 Subject: [PATCH 339/634] #11772 more tests for public install --- .../tests/data/update-dataset-terms.json | 4 ++-- .../harvard/iq/dataverse/api/DatasetsIT.java | 9 +++++++++ .../json/update-dataset-terms-access.json | 2 +- .../json/update-dataset-terms-no-access.json | 20 +++++++++++++++++++ 4 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 src/test/resources/json/update-dataset-terms-no-access.json diff --git a/scripts/search/tests/data/update-dataset-terms.json b/scripts/search/tests/data/update-dataset-terms.json index dbe6380ca56..b8b366f5927 100644 --- a/scripts/search/tests/data/update-dataset-terms.json +++ b/scripts/search/tests/data/update-dataset-terms.json @@ -1,7 +1,7 @@ { "termsOfAccess": { - "fileAccessRequest": "true", - "termsOfAccessForRestrictedFiles": "For access to restriced files please see read me file", + "fileAccessRequest": "false", + "termsOfAccess": "", "dataAccessPlace": "dataAccessPlace", "originalArchive": "originalArchive", "availabilityStatus": "availabilityStatus", diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 1a128c14bee..242c2de4aae 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -4189,6 +4189,15 @@ public void testUpdateDatasetTerms() throws IOException { .body("message", equalTo("Setting File Access Request or Terms of Access is not permitted on a public installation.")); //Setting File Access Request or Terms of Access is not permitted on a public installation. + + pathToJsonFile = "src/test/resources/json/update-dataset-terms-no-access.json"; + + updateTerms = UtilIT.updateDatasetTermsAndAccess(datasetPid, apiToken, pathToJsonFile); + updateTerms.prettyPrint(); + // should be ok because it doesn't update request or terms of access + updateTerms.then().assertThat() + .statusCode(OK.getStatusCode()); + //reset public install UtilIT.setSetting(SettingsServiceBean.Key.PublicInstall, publicInstall); diff --git a/src/test/resources/json/update-dataset-terms-access.json b/src/test/resources/json/update-dataset-terms-access.json index 29749cd21dd..84fe4e1f46f 100644 --- a/src/test/resources/json/update-dataset-terms-access.json +++ b/src/test/resources/json/update-dataset-terms-access.json @@ -1,6 +1,6 @@ { "termsOfUseAndAccess": { - "fileAccessRequest": "false", + "fileAccessRequest": "true", "termsOfAccess": "For access to restriced files please see read me file", "dataAccessPlace": "dataAccessPlace", "originalArchive": "originalArchive", diff --git a/src/test/resources/json/update-dataset-terms-no-access.json b/src/test/resources/json/update-dataset-terms-no-access.json new file mode 100644 index 00000000000..8cf8d7cb554 --- /dev/null +++ b/src/test/resources/json/update-dataset-terms-no-access.json @@ -0,0 +1,20 @@ +{ + "termsOfUseAndAccess": { + "fileAccessRequest": "false", + "termsOfAccess": "", + "dataAccessPlace": "dataAccessPlace", + "originalArchive": "originalArchive", + "availabilityStatus": "availabilityStatus", + "contactForAccess": "contactForAccess", + "sizeOfCollection": "sizeOfCollection", + "studyCompletion": "studyCompletion", + "termsOfUse": "termsOfUse", + "confidentialityDeclaration": "confidentialityDeclaration", + "specialPermissions": "specialPermissions", + "restrictions": "restrictions", + "citationRequirements": "citationRequirements", + "depositorRequirements": "depositorRequirements", + "conditions": "conditions", + "disclaimer": "disclaimer" + } +} From eeb6e33bb893a92f6c3876ce050cd4c5d499649b Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 2 Oct 2025 14:41:07 +0100 Subject: [PATCH 340/634] Refactor: getCompareVersionsSummary and related layers --- .../harvard/iq/dataverse/api/Datasets.java | 72 +------------------ .../DatasetVersionSummary.java | 7 +- ...tasetVersionSummaryContentDifferences.java | 12 ++++ .../DatasetVersionSummaryContentSimple.java | 6 +- .../GetDatasetVersionSummariesCommand.java | 2 + .../iq/dataverse/util/json/JsonPrinter.java | 45 ++++++++++-- 6 files changed, 65 insertions(+), 79 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index b2a93f7b2b0..db09768e879 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -2,7 +2,6 @@ import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.DatasetLock.Reason; -import edu.harvard.iq.dataverse.DatasetVersion.VersionState; import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.api.dto.RoleAssignmentDTO; @@ -3141,80 +3140,11 @@ public Response getCompareVersions(@Context ContainerRequestContext crc, @PathPa public Response getCompareVersionsSummary(@Context ContainerRequestContext crc, @PathParam("id") String id) { return response(req -> { try { - return ok(json(execCommand(new GetDatasetVersionSummariesCommand(req, findDatasetOrDie(id))))); + return ok(jsonDatasetVersionSummaries(execCommand(new GetDatasetVersionSummariesCommand(req, findDatasetOrDie(id))))); } catch (WrappedResponse wr) { return wr.getResponse(); } }, getRequestUser(crc)); - /* - try { - Dataset dataset = findDatasetOrDie(id); - User user = getRequestUser(crc); - JsonArrayBuilder differenceSummaries = Json.createArrayBuilder(); - - for (DatasetVersion dv : dataset.getVersions()) { - //only get summaries of draft is user may view unpublished - - if (dv.isPublished() ||dv.isDeaccessioned() || permissionService.hasPermissionsFor(user, dv.getDataset(), - EnumSet.of(Permission.ViewUnpublishedDataset))) { - - JsonObjectBuilder versionBuilder = new NullSafeJsonBuilder(); - versionBuilder.add("id", dv.getId()); - versionBuilder.add("versionNumber", dv.getFriendlyVersionNumber()); - DatasetVersionDifference dvdiff = dv.getDefaultVersionDifference(); - if (dvdiff == null) { - if (dv.isReleased()) { - if (dv.getPriorVersionState() == null) { - versionBuilder.add("summary", "firstPublished"); - } - if (dv.getPriorVersionState() != null && dv.getPriorVersionState().equals(VersionState.DEACCESSIONED)) { - versionBuilder.add("summary", "previousVersionDeaccessioned"); - } - } - if (dv.isDraft()) { - if (dv.getPriorVersionState() == null) { - versionBuilder.add("summary", "firstDraft"); - } - if (dv.getPriorVersionState() != null && dv.getPriorVersionState().equals(VersionState.DEACCESSIONED)) { - versionBuilder.add("summary", "previousVersionDeaccessioned"); - } - } - if (dv.isDeaccessioned()) { - versionBuilder.add("summary", getDeaccessionJson(dv)); - } - - } else { - versionBuilder.add("summary", dvdiff.getSummaryDifferenceAsJson()); - } - versionBuilder.add("versionNote", dv.getVersionNote()); - versionBuilder.add("contributors", datasetversionService.getContributorsNames(dv)); - versionBuilder.add("publishedOn", !dv.isDraft() ? dv.getPublicationDateAsString() : ""); - differenceSummaries.add(versionBuilder); - } - } - return ok(differenceSummaries); - } catch (WrappedResponse wr) { - return wr.getResponse(); - }*/ - } - - private JsonObject getDeaccessionJson(DatasetVersion dv) { - - JsonObjectBuilder compositionBuilder = Json.createObjectBuilder(); - - if (dv.getDeaccessionNote() != null && !dv.getDeaccessionNote().isEmpty()) { - compositionBuilder.add("reason", dv.getDeaccessionNote()); - } - - if (dv.getDeaccessionLink() != null && !dv.getDeaccessionLink().isEmpty()) { - compositionBuilder.add("url", dv.getDeaccessionLink()); - } - - JsonObject json = Json.createObjectBuilder() - .add("deaccessioned", compositionBuilder) - .build(); - - return json; } private static Set getDatasetFilenames(Dataset dataset) { diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummary.java b/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummary.java index 4a34e406398..06da005166f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummary.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummary.java @@ -10,7 +10,8 @@ * An immutable data carrier representing a summary of a DatasetVersion. */ public record DatasetVersionSummary( - String versionId, + Long id, + String versionNumber, String versionNote, String contributorNames, String publicationDate, @@ -28,6 +29,7 @@ public record DatasetVersionSummary( */ public static Optional from(DatasetVersion datasetVersion) { return Optional.ofNullable(datasetVersion).map(version -> new DatasetVersionSummary( + version.getId(), version.getFriendlyVersionNumber(), version.getVersionNote(), version.getContributorNames(), @@ -46,8 +48,7 @@ public static Optional from(DatasetVersion datasetVersion private static DatasetVersionSummaryContent determineContent(DatasetVersion version) { // Priority 1: The version has explicit differences calculated. if (version.getDefaultVersionDifference() != null) { - // TODO - return new DatasetVersionSummaryContentDifferences(); + return new DatasetVersionSummaryContentDifferences(version.getDefaultVersionDifference()); } // Priority 2: Deaccessioned status is a critical override. diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentDifferences.java b/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentDifferences.java index c20eebfc400..57add5511da 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentDifferences.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentDifferences.java @@ -1,4 +1,16 @@ package edu.harvard.iq.dataverse.datasetversionsummaries; +import edu.harvard.iq.dataverse.DatasetVersionDifference; + public class DatasetVersionSummaryContentDifferences extends DatasetVersionSummaryContent { + + private final DatasetVersionDifference datasetVersionDifference; + + public DatasetVersionSummaryContentDifferences(DatasetVersionDifference datasetVersionDifference) { + this.datasetVersionDifference = datasetVersionDifference; + } + + public DatasetVersionDifference getDatasetVersionDifference() { + return datasetVersionDifference; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentSimple.java b/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentSimple.java index db0172349f6..5d1075b41e6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentSimple.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryContentSimple.java @@ -2,12 +2,16 @@ public class DatasetVersionSummaryContentSimple extends DatasetVersionSummaryContent { - private Content value; + private final Content value; public DatasetVersionSummaryContentSimple(Content value) { this.value = value; } + public Content getValue() { + return value; + } + public enum Content { firstPublished, previousVersionDeaccessioned, diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommand.java index 23ea0658f0a..156d5b4cabd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommand.java @@ -7,12 +7,14 @@ import edu.harvard.iq.dataverse.engine.command.AbstractCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import java.util.EnumSet; import java.util.List; import java.util.Optional; +@RequiredPermissions({}) public class GetDatasetVersionSummariesCommand extends AbstractCommand> { private final Dataset dataset; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index c6d3a18be87..49d887cf27d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -19,7 +19,7 @@ import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataset.DatasetType; import edu.harvard.iq.dataverse.dataset.DatasetUtil; -import edu.harvard.iq.dataverse.datasetversionsummaries.DatasetVersionSummary; +import edu.harvard.iq.dataverse.datasetversionsummaries.*; import edu.harvard.iq.dataverse.datavariable.CategoryMetadata; import edu.harvard.iq.dataverse.datavariable.DataVariable; import edu.harvard.iq.dataverse.datavariable.SummaryStatistic; @@ -1687,8 +1687,45 @@ public static JsonArrayBuilder json(List notifications, Authen return notificationsArray; } - public static JsonArrayBuilder json(List datasetVersionSummaries) { - // TODO - return Json.createArrayBuilder(); + public static JsonArray jsonDatasetVersionSummaries(List summaries) { + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + summaries.stream() + .filter(Objects::nonNull) + .map(JsonPrinter::json) + .forEach(arrayBuilder::add); + + return arrayBuilder.build(); + } + + private static JsonObjectBuilder json(DatasetVersionSummary summary) { + JsonObjectBuilder jsonObjectBuilder = jsonObjectBuilder() + .add("id", summary.id()) + .add("versionNumber", summary.versionNumber()) + .add("versionNote", summary.versionNote()) + .add("contributors", summary.contributorNames()) + .add("publishedOn", summary.publicationDate()); + + DatasetVersionSummaryContent content = summary.content(); + if (content instanceof DatasetVersionSummaryContentDeaccessioned deaccessioned) { + JsonObject deaccessionedDetailsJsonObject = jsonObjectBuilder() + .add("reason", deaccessioned.getDeaccessionNote()) + .add("url", deaccessioned.getDeaccessionLink()) + .build(); + JsonObject deaccessionedJsonObject = jsonObjectBuilder() + .add("deaccessioned", deaccessionedDetailsJsonObject) + .build(); + jsonObjectBuilder.add("summary", deaccessionedJsonObject + ); + } else if (content instanceof DatasetVersionSummaryContentSimple simple) { + jsonObjectBuilder.add("summary", simple.getValue().name()); + + } else if (content instanceof DatasetVersionSummaryContentDifferences differences) { + jsonObjectBuilder.add("summary", differences + .getDatasetVersionDifference() + .getSummaryDifferenceAsJson() + .build()); + } + + return jsonObjectBuilder; } } From fb5f9115fc588ab74f2df44b13f6ce702dae751b Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 2 Oct 2025 09:51:48 -0400 Subject: [PATCH 341/634] #11772 remove unused file --- .../tests/data/update-dataset-terms.json | 20 ------------------- 1 file changed, 20 deletions(-) delete mode 100644 scripts/search/tests/data/update-dataset-terms.json diff --git a/scripts/search/tests/data/update-dataset-terms.json b/scripts/search/tests/data/update-dataset-terms.json deleted file mode 100644 index b8b366f5927..00000000000 --- a/scripts/search/tests/data/update-dataset-terms.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "termsOfAccess": { - "fileAccessRequest": "false", - "termsOfAccess": "", - "dataAccessPlace": "dataAccessPlace", - "originalArchive": "originalArchive", - "availabilityStatus": "availabilityStatus", - "contactForAccess": "contactForAccess", - "sizeOfCollection": "sizeOfCollection", - "studyCompletion": "studyCompletion", - "termsOfUse": "termsOfUse", - "confidentialityDeclaration": "confidentialityDeclaration", - "specialPermissions": "specialPermissions", - "restrictions": "restrictions", - "citationRequirements": "citationRequirements", - "depositorRequirements": "depositorRequirements", - "conditions": "conditions", - "disclaimer": "disclaimer" - } -} From dda50b8731edebdf02a44e5990f57a507bcf15d9 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 2 Oct 2025 10:37:35 -0400 Subject: [PATCH 342/634] #11772 fix json/tests --- .../harvard/iq/dataverse/api/DatasetsIT.java | 39 +++++++++---------- .../json/update-dataset-terms-access.json | 4 +- .../json/update-dataset-terms-no-access.json | 2 +- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 242c2de4aae..1faec141512 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -4094,7 +4094,7 @@ public void testFilesUnchangedAfterDatasetMetadataUpdate() throws IOException { @Test public void testUpdateDatasetTerms() throws IOException { - + //get publicInstall setting so we can change it back Response publicInstallResponse = UtilIT.getSetting(SettingsServiceBean.Key.PublicInstall); //TODO: fix this its a little hacky @@ -4102,7 +4102,7 @@ public void testUpdateDatasetTerms() throws IOException { // make sure this is not a public installation UtilIT.setSetting(SettingsServiceBean.Key.PublicInstall, "false"); - + Response createUser = UtilIT.createRandomUser(); createUser.prettyPrint(); createUser.then().assertThat() @@ -4139,9 +4139,7 @@ public void testUpdateDatasetTerms() throws IOException { .statusCode(OK.getStatusCode()); //Add another dataset with restricted files in order to get the access info to show up - - - Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode()); int datasetId2 = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); String datasetPersistentId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId"); @@ -4149,22 +4147,23 @@ public void testUpdateDatasetTerms() throws IOException { updateTerms = UtilIT.updateDatasetTermsAndAccess(datasetPersistentId, apiToken, pathToJsonFile); updateTerms.prettyPrint(); updateTerms.then().assertThat() - .statusCode(OK.getStatusCode()); - - // Upload file + .statusCode(OK.getStatusCode()) + .body("data.fileAccessRequest", equalTo(true)); + String pathToTestFile = "src/test/resources/images/coffeeshop.png"; Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId2), pathToTestFile, Json.createObjectBuilder().build(), apiToken); uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); String fileId = JsonPath.from(uploadResponse.body().asString()).getString("data.files[0].dataFile.id"); - + updateTerms = UtilIT.updateDatasetTermsAndAccess(datasetPersistentId, apiToken, pathToJsonFile); updateTerms.prettyPrint(); updateTerms.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data.termsOfAccess", equalTo("For access to restriced files please see read me file")); + .body("data.fileAccessRequest", equalTo(true)) + .body("data.termsOfAccess", equalTo("For access to restricted files please see read me file")); - // Restrict file + // Restrict file Response restrictFileResponse = UtilIT.restrictFile(fileId, true, apiToken); restrictFileResponse.then().assertThat().statusCode(OK.getStatusCode()); @@ -4172,15 +4171,15 @@ public void testUpdateDatasetTerms() throws IOException { Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetPersistentId, "major", apiToken); publishDatasetResponse.prettyPrint(); publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode()); - + //Make installation "public install" to see that terms of access may not be set UtilIT.setSetting(SettingsServiceBean.Key.PublicInstall, "true"); - + createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); createDataset.prettyPrint(); createDataset.then().assertThat() .statusCode(CREATED.getStatusCode()); - + updateTerms = UtilIT.updateDatasetTermsAndAccess(datasetPid, apiToken, pathToJsonFile); updateTerms.prettyPrint(); // should fail because its now a public install @@ -4188,16 +4187,16 @@ public void testUpdateDatasetTerms() throws IOException { .statusCode(BAD_REQUEST.getStatusCode()) .body("message", equalTo("Setting File Access Request or Terms of Access is not permitted on a public installation.")); //Setting File Access Request or Terms of Access is not permitted on a public installation. - - - pathToJsonFile = "src/test/resources/json/update-dataset-terms-no-access.json"; - + + pathToJsonFile = "src/test/resources/json/update-dataset-terms-no-access.json"; + updateTerms = UtilIT.updateDatasetTermsAndAccess(datasetPid, apiToken, pathToJsonFile); updateTerms.prettyPrint(); // should be ok because it doesn't update request or terms of access updateTerms.then().assertThat() - .statusCode(OK.getStatusCode()); - + .statusCode(OK.getStatusCode()) + .body("data.fileAccessRequest", equalTo(false)); + //reset public install UtilIT.setSetting(SettingsServiceBean.Key.PublicInstall, publicInstall); diff --git a/src/test/resources/json/update-dataset-terms-access.json b/src/test/resources/json/update-dataset-terms-access.json index 84fe4e1f46f..fbbd45a313f 100644 --- a/src/test/resources/json/update-dataset-terms-access.json +++ b/src/test/resources/json/update-dataset-terms-access.json @@ -1,7 +1,7 @@ { "termsOfUseAndAccess": { - "fileAccessRequest": "true", - "termsOfAccess": "For access to restriced files please see read me file", + "fileAccessRequest": true, + "termsOfAccess": "For access to restricted files please see read me file", "dataAccessPlace": "dataAccessPlace", "originalArchive": "originalArchive", "availabilityStatus": "availabilityStatus", diff --git a/src/test/resources/json/update-dataset-terms-no-access.json b/src/test/resources/json/update-dataset-terms-no-access.json index 8cf8d7cb554..9c53330b7c5 100644 --- a/src/test/resources/json/update-dataset-terms-no-access.json +++ b/src/test/resources/json/update-dataset-terms-no-access.json @@ -1,6 +1,6 @@ { "termsOfUseAndAccess": { - "fileAccessRequest": "false", + "fileAccessRequest": false, "termsOfAccess": "", "dataAccessPlace": "dataAccessPlace", "originalArchive": "originalArchive", From 98c72cbfbe67ebe13d25b6ec65b49f05924eba54 Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 2 Oct 2025 18:51:23 +0100 Subject: [PATCH 343/634] Changed: GetDatasetVersionSummariesCommand using DatasetVersionServiceBean.findVersions --- .../harvard/iq/dataverse/DatasetVersion.java | 6 +- .../dataverse/DatasetVersionServiceBean.java | 67 +++++++++++++------ .../GetDatasetVersionSummariesCommand.java | 23 ++++--- 3 files changed, 66 insertions(+), 30 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index 14ddf65a833..cfd4c886bf3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -73,7 +73,11 @@ @NamedQuery(name = "DatasetVersion.findById", query = "SELECT o FROM DatasetVersion o LEFT JOIN FETCH o.fileMetadatas WHERE o.id=:id"), @NamedQuery(name = "DatasetVersion.findByDataset", - query = "SELECT o FROM DatasetVersion o WHERE o.dataset.id=:datasetId ORDER BY o.versionNumber DESC, o.minorVersionNumber DESC"), + query = "SELECT o FROM DatasetVersion o WHERE o.dataset.id=:datasetId ORDER BY o.versionNumber DESC, o.minorVersionNumber DESC"), + @NamedQuery(name = "DatasetVersion.findByDesiredStatesAndDataset", + query = "SELECT o FROM DatasetVersion o " + + "WHERE o.dataset.id = :datasetId AND o.versionState IN :states " + + "ORDER BY o.versionNumber DESC, o.minorVersionNumber DESC"), @NamedQuery(name = "DatasetVersion.findReleasedByDataset", query = "SELECT o FROM DatasetVersion o WHERE o.dataset.id=:datasetId AND o.versionState=edu.harvard.iq.dataverse.DatasetVersion.VersionState.RELEASED ORDER BY o.versionNumber DESC, o.minorVersionNumber DESC")/*, @NamedQuery(name = "DatasetVersion.findVersionElements", diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java index 7e9b778c6f3..0a60051ea10 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java @@ -171,41 +171,66 @@ public DatasetVersion findDeep(Object pk) { .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.dataFile.dataFileTags") .getSingleResult(); } - + /** - * Performs the same database lookup as the one behind Dataset.getVersions(). - * Additionally, provides the arguments for selecting a partial list of - * (length-offset) versions for pagination, plus the ability to pre-select - * only the publicly-viewable versions. - * It is recommended that individual software components utilize the - * ListVersionsCommand, instead of calling this service method directly. - * @param datasetId - * @param offset for pagination through long lists of versions - * @param length for pagination through long lists of versions - * @param includeUnpublished retrieves all the versions, including drafts and deaccessioned. - * @return (partial) list of versions + * Performs the same database lookup as the one behind {@code Dataset.getVersions()}. + *

      + * Additionally, supports: + *

        + *
      • Pagination via {@code offset} and {@code length}
        • + *
      • Filtering by visibility (all, released only, or released + deaccessioned)
        • + *
      + *

      + * It is recommended that individual software components utilize + * {@link edu.harvard.iq.dataverse.engine.command.impl.ListVersionsCommand}, + * instead of calling this service method directly. + * + * @param datasetId the dataset identifier + * @param offset pagination offset (nullable) + * @param length pagination length (nullable) + * @param includeAllVersions if {@code true}, retrieves all versions (drafts, released, and deaccessioned) + * @param includeDeaccessioned if {@code true}, includes deaccessioned versions + * when {@code includeAll} is {@code false} + * @return a (possibly partial) list of dataset versions */ - public List findVersions(Long datasetId, Integer offset, Integer length, boolean includeUnpublished) { - TypedQuery query; - if (includeUnpublished) { + public List findVersions(Long datasetId, + Integer offset, + Integer length, + boolean includeAllVersions, + boolean includeDeaccessioned) { + TypedQuery query; + + if (includeAllVersions) { query = em.createNamedQuery("DatasetVersion.findByDataset", DatasetVersion.class); + } else if (includeDeaccessioned) { + query = em.createNamedQuery("DatasetVersion.findByDesiredStatesAndDataset", DatasetVersion.class); + query.setParameter("states", List.of( + VersionState.RELEASED, + VersionState.DEACCESSIONED + )); } else { - query = em.createNamedQuery("DatasetVersion.findReleasedByDataset", DatasetVersion.class) - .setParameter("datasetId", datasetId); + query = em.createNamedQuery("DatasetVersion.findReleasedByDataset", DatasetVersion.class); } - + query.setParameter("datasetId", datasetId); - + if (offset != null) { query.setFirstResult(offset); } if (length != null) { query.setMaxResults(length); } - + return query.getResultList(); } - + + public List findVersions(Long datasetId, + Integer offset, + Integer length, + boolean includeAllVersions) { + return findVersions(datasetId, offset, length, includeAllVersions, false); + } + public DatasetVersion findByFriendlyVersionNumber(Long datasetId, String friendlyVersionNumber) { Long majorVersionNumber = null; Long minorVersionNumber = null; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommand.java index 156d5b4cabd..baa0da172f4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommand.java @@ -26,16 +26,23 @@ public GetDatasetVersionSummariesCommand(DataverseRequest request, Dataset datas @Override public List execute(CommandContext ctxt) throws CommandException { - return dataset.getVersions().stream() - .filter(dv -> versionRequiresSummary(ctxt, dv)) + boolean canViewUnpublished = ctxt.permissions().hasPermissionsFor( + getRequest().getUser(), + dataset, + EnumSet.of(Permission.ViewUnpublishedDataset) + ); + + List versions = ctxt.datasetVersion().findVersions( + dataset.getId(), + null, + null, + canViewUnpublished, + true + ); + + return versions.stream() .map(DatasetVersionSummary::from) .flatMap(Optional::stream) .toList(); } - - private boolean versionRequiresSummary(CommandContext ctxt, DatasetVersion dv) { - return dv.isPublished() - || dv.isDeaccessioned() - || ctxt.permissions().hasPermissionsFor(getRequest().getUser(), dv.getDataset(), EnumSet.of(Permission.ViewUnpublishedDataset)); - } } From 5919cf479272c62684dc27ada15ab375c09ca866 Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 2 Oct 2025 19:01:28 +0100 Subject: [PATCH 344/634] Added: handling pagination optional params on getCompareVersionsSummary API endpoint --- .../java/edu/harvard/iq/dataverse/api/Datasets.java | 7 +++++-- .../impl/GetDatasetVersionSummariesCommand.java | 13 ++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index db09768e879..c2888aa0f65 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3137,10 +3137,13 @@ public Response getCompareVersions(@Context ContainerRequestContext crc, @PathPa @GET @AuthRequired @Path("{id}/versions/compareSummary") - public Response getCompareVersionsSummary(@Context ContainerRequestContext crc, @PathParam("id") String id) { + public Response getCompareVersionsSummary(@Context ContainerRequestContext crc, + @PathParam("id") String id, + @QueryParam("limit") Integer limit, + @QueryParam("offset") Integer offset) { return response(req -> { try { - return ok(jsonDatasetVersionSummaries(execCommand(new GetDatasetVersionSummariesCommand(req, findDatasetOrDie(id))))); + return ok(jsonDatasetVersionSummaries(execCommand(new GetDatasetVersionSummariesCommand(req, findDatasetOrDie(id), limit, offset)))); } catch (WrappedResponse wr) { return wr.getResponse(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommand.java index baa0da172f4..d820dd7de6a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommand.java @@ -14,14 +14,21 @@ import java.util.List; import java.util.Optional; +/** + * A command that retrieves a paginated list of {@link DatasetVersionSummary} for the versions of a given {@link Dataset}. + **/ @RequiredPermissions({}) public class GetDatasetVersionSummariesCommand extends AbstractCommand> { private final Dataset dataset; + private final Integer limit; + private final Integer offset; - public GetDatasetVersionSummariesCommand(DataverseRequest request, Dataset dataset) { + public GetDatasetVersionSummariesCommand(DataverseRequest request, Dataset dataset, Integer limit, Integer offset) { super(request, dataset); this.dataset = dataset; + this.limit = limit; + this.offset = offset; } @Override @@ -34,8 +41,8 @@ public List execute(CommandContext ctxt) throws CommandEx List versions = ctxt.datasetVersion().findVersions( dataset.getId(), - null, - null, + offset, + limit, canViewUnpublished, true ); From ea5642e33d545f3829660aeaf05578bd05b9e1c9 Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 2 Oct 2025 19:14:50 +0100 Subject: [PATCH 345/634] Added: DatasetVersionSummaryTest --- .../DatasetVersionSummaryTest.java | 195 ++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 src/test/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryTest.java diff --git a/src/test/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryTest.java b/src/test/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryTest.java new file mode 100644 index 00000000000..ba85ff640f4 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryTest.java @@ -0,0 +1,195 @@ +package edu.harvard.iq.dataverse.datasetversionsummaries; + +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.DatasetVersion.VersionState; +import edu.harvard.iq.dataverse.DatasetVersionDifference; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +/** + * Unit tests for the {@link DatasetVersionSummary} record and its factory methods. + */ +@ExtendWith(MockitoExtension.class) +class DatasetVersionSummaryTest { + + @Mock + private DatasetVersion versionMock; + + @Mock + private DatasetVersionDifference differenceMock; + + @Test + @DisplayName("from() should return an empty Optional when the input DatasetVersion is null") + void from_whenVersionIsNull_shouldReturnEmptyOptional() { + // Act + Optional result = DatasetVersionSummary.from(null); + + // Assert + assertThat(result).isNotPresent(); + } + + @Test + @DisplayName("from() should map all basic properties from the DatasetVersion correctly") + void from_whenVersionIsNotNull_shouldMapPropertiesCorrectly() { + // Arrange + when(versionMock.getId()).thenReturn(42L); + when(versionMock.getFriendlyVersionNumber()).thenReturn("V1.0"); + when(versionMock.getVersionNote()).thenReturn("Initial version note."); + when(versionMock.getContributorNames()).thenReturn("Contributor One, Contributor Two"); + when(versionMock.getPublicationDateAsString()).thenReturn("2025-10-02"); + when(versionMock.isReleased()).thenReturn(true); + when(versionMock.getPriorVersionState()).thenReturn(null); + + // Act + Optional summaryOpt = DatasetVersionSummary.from(versionMock); + + // Assert + assertThat(summaryOpt).isPresent(); + DatasetVersionSummary summary = summaryOpt.get(); + + assertThat(summary.id()).isEqualTo(42L); + assertThat(summary.versionNumber()).isEqualTo("V1.0"); + assertThat(summary.versionNote()).isEqualTo("Initial version note."); + assertThat(summary.contributorNames()).isEqualTo("Contributor One, Contributor Two"); + assertThat(summary.publicationDate()).isEqualTo("2025-10-02"); + } + + @Test + @DisplayName("Content should be 'Differences' if a version difference is present (highest priority)") + void from_whenVersionHasDifferences_shouldCreateDifferencesContent() { + // Arrange + when(versionMock.getDefaultVersionDifference()).thenReturn(differenceMock); + + // Act + Optional summaryOpt = DatasetVersionSummary.from(versionMock); + + // Assert + assertThat(summaryOpt).isPresent(); + assertThat(summaryOpt.get().content()).isInstanceOf(DatasetVersionSummaryContentDifferences.class); + DatasetVersionSummaryContentDifferences content = (DatasetVersionSummaryContentDifferences) summaryOpt.get().content(); + assertThat(content.getDatasetVersionDifference()).isSameAs(differenceMock); + } + + @Test + @DisplayName("Content should be 'Deaccessioned' if the version is deaccessioned") + void from_whenVersionIsDeaccessioned_shouldCreateDeaccessionedContent() { + // Arrange + when(versionMock.getDefaultVersionDifference()).thenReturn(null); // No differences + when(versionMock.isDeaccessioned()).thenReturn(true); + when(versionMock.getDeaccessionNote()).thenReturn("Reason for deaccession."); + when(versionMock.getDeaccessionLink()).thenReturn("http://example.com/deaccession"); + + // Act + Optional summaryOpt = DatasetVersionSummary.from(versionMock); + + // Assert + assertThat(summaryOpt).isPresent(); + DatasetVersionSummaryContentDeaccessioned content = (DatasetVersionSummaryContentDeaccessioned) summaryOpt.get().content(); + + assertThat(content.getDeaccessionNote()).isEqualTo("Reason for deaccession."); + assertThat(content.getDeaccessionLink()).isEqualTo("http://example.com/deaccession"); + } + + @Test + @DisplayName("Content should be 'firstPublished' for the first released version") + void from_whenIsFirstPublishedVersion_shouldCreateFirstPublishedContent() { + // Arrange + when(versionMock.getDefaultVersionDifference()).thenReturn(null); + when(versionMock.isDeaccessioned()).thenReturn(false); + when(versionMock.isReleased()).thenReturn(true); + when(versionMock.getPriorVersionState()).thenReturn(null); // This makes it the "first" version + + // Act + Optional summaryOpt = DatasetVersionSummary.from(versionMock); + + // Assert + assertThat(summaryOpt).isPresent(); + assertThat(summaryOpt.get().content()) + .isInstanceOf(DatasetVersionSummaryContentSimple.class) + .extracting("value") + .isEqualTo(DatasetVersionSummaryContentSimple.Content.firstPublished); + } + + @Test + @DisplayName("Content should be 'firstDraft' for the first draft version") + void from_whenIsFirstDraftVersion_shouldCreateFirstDraftContent() { + // Arrange + when(versionMock.getDefaultVersionDifference()).thenReturn(null); + when(versionMock.isDeaccessioned()).thenReturn(false); + when(versionMock.isDraft()).thenReturn(true); + when(versionMock.getPriorVersionState()).thenReturn(null); // This makes it the "first" version + + // Act + Optional summaryOpt = DatasetVersionSummary.from(versionMock); + + // Assert + assertThat(summaryOpt).isPresent(); + assertThat(summaryOpt.get().content()) + .isInstanceOf(DatasetVersionSummaryContentSimple.class) + .extracting("value") + .isEqualTo(DatasetVersionSummaryContentSimple.Content.firstDraft); + } + + @Test + @DisplayName("Content should be 'previousVersionDeaccessioned' if the prior version was deaccessioned") + void from_whenPriorVersionWasDeaccessioned_shouldCreateAppropriateContent() { + // Arrange + when(versionMock.getDefaultVersionDifference()).thenReturn(null); + when(versionMock.isDeaccessioned()).thenReturn(false); + when(versionMock.isReleased()).thenReturn(true); + when(versionMock.getPriorVersionState()).thenReturn(VersionState.DEACCESSIONED); + + // Act + Optional summaryOpt = DatasetVersionSummary.from(versionMock); + + // Assert + assertThat(summaryOpt).isPresent(); + assertThat(summaryOpt.get().content()) + .isInstanceOf(DatasetVersionSummaryContentSimple.class) + .extracting("value") + .isEqualTo(DatasetVersionSummaryContentSimple.Content.previousVersionDeaccessioned); + } + + @Test + @DisplayName("Content should be null for a standard released version that is not the first") + void from_whenIsStandardUpdate_shouldHaveNullContent() { + // Arrange + when(versionMock.getDefaultVersionDifference()).thenReturn(null); + when(versionMock.isDeaccessioned()).thenReturn(false); + when(versionMock.isReleased()).thenReturn(true); + when(versionMock.getPriorVersionState()).thenReturn(VersionState.RELEASED); // Prior version exists + + // Act + Optional summaryOpt = DatasetVersionSummary.from(versionMock); + + // Assert + assertThat(summaryOpt).isPresent(); + assertThat(summaryOpt.get().content()).isNull(); + } + + @Test + @DisplayName("Content should be null for an unhandled version state like ARCHIVED") + void from_whenStateIsUnhandled_shouldHaveNullContent() { + // Arrange + when(versionMock.getDefaultVersionDifference()).thenReturn(null); + when(versionMock.isDeaccessioned()).thenReturn(false); + when(versionMock.isReleased()).thenReturn(false); + when(versionMock.isDraft()).thenReturn(false); + // Assuming VersionState is ARCHIVED, isArchived() would be true but the logic doesn't check for it. + + // Act + Optional summaryOpt = DatasetVersionSummary.from(versionMock); + + // Assert + assertThat(summaryOpt).isPresent(); + assertThat(summaryOpt.get().content()).isNull(); + } +} \ No newline at end of file From b3c4fbdb19c4780b05f5c63f8f8306608c270a69 Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 2 Oct 2025 19:21:05 +0100 Subject: [PATCH 346/634] Added: GetDatasetVersionSummariesCommandTest --- .../DatasetVersionSummaryTest.java | 2 +- ...GetDatasetVersionSummariesCommandTest.java | 172 ++++++++++++++++++ 2 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 src/test/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommandTest.java diff --git a/src/test/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryTest.java b/src/test/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryTest.java index ba85ff640f4..430c945591c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/datasetversionsummaries/DatasetVersionSummaryTest.java @@ -192,4 +192,4 @@ void from_whenStateIsUnhandled_shouldHaveNullContent() { assertThat(summaryOpt).isPresent(); assertThat(summaryOpt.get().content()).isNull(); } -} \ No newline at end of file +} diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommandTest.java new file mode 100644 index 00000000000..0161bdbfbf2 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetVersionSummariesCommandTest.java @@ -0,0 +1,172 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.DatasetVersionServiceBean; +import edu.harvard.iq.dataverse.PermissionServiceBean; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.datasetversionsummaries.DatasetVersionSummary; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class GetDatasetVersionSummariesCommandTest { + + @Mock + private CommandContext contextMock; + @Mock + private DataverseRequest requestMock; + @Mock + private Dataset datasetMock; + @Mock + private PermissionServiceBean permissionsMock; + @Mock + private DatasetVersionServiceBean versionServiceMock; + @Mock + private AuthenticatedUser userMock; + @Mock + private DatasetVersion versionMock; + + private static final Long DATASET_ID = 42L; + + @BeforeEach + void setUp() { + // Mock the context to return our mock services + when(contextMock.permissions()).thenReturn(permissionsMock); + when(contextMock.datasetVersion()).thenReturn(versionServiceMock); + + // Mock the request to return a user + when(requestMock.getUser()).thenReturn(userMock); + + // Mock the dataset to have a specific ID + when(datasetMock.getId()).thenReturn(DATASET_ID); + } + + @Test + @DisplayName("execute should call findVersions with 'canViewUnpublished' as TRUE when user has permission") + void execute_should_call_findVersions_with_true_when_user_has_permission() throws CommandException { + // Arrange + when(permissionsMock.hasPermissionsFor(requestMock.getUser(), datasetMock, EnumSet.of(Permission.ViewUnpublishedDataset))) + .thenReturn(true); + when(versionServiceMock.findVersions(anyLong(), any(), any(), anyBoolean(), anyBoolean())) + .thenReturn(Collections.emptyList()); + + GetDatasetVersionSummariesCommand command = new GetDatasetVersionSummariesCommand(requestMock, datasetMock, null, null); + + // Act + command.execute(contextMock); + + // Assert + ArgumentCaptor canViewUnpublishedCaptor = ArgumentCaptor.forClass(Boolean.class); + verify(versionServiceMock).findVersions( + eq(DATASET_ID), + eq(null), + eq(null), + canViewUnpublishedCaptor.capture(), + eq(true) + ); + assertThat(canViewUnpublishedCaptor.getValue()).isTrue(); + } + + @Test + @DisplayName("execute should call findVersions with 'canViewUnpublished' as FALSE when user lacks permission") + void execute_should_call_findVersions_with_false_when_user_lacks_permission() throws CommandException { + // Arrange + when(permissionsMock.hasPermissionsFor(requestMock.getUser(), datasetMock, EnumSet.of(Permission.ViewUnpublishedDataset))) + .thenReturn(false); + when(versionServiceMock.findVersions(anyLong(), any(), any(), anyBoolean(), anyBoolean())) + .thenReturn(Collections.emptyList()); + + GetDatasetVersionSummariesCommand command = new GetDatasetVersionSummariesCommand(requestMock, datasetMock, null, null); + + // Act + command.execute(contextMock); + + // Assert + ArgumentCaptor canViewUnpublishedCaptor = ArgumentCaptor.forClass(Boolean.class); + verify(versionServiceMock).findVersions( + eq(DATASET_ID), + eq(null), + eq(null), + canViewUnpublishedCaptor.capture(), + eq(true) + ); + assertThat(canViewUnpublishedCaptor.getValue()).isFalse(); + } + + @Test + @DisplayName("execute should pass pagination parameters correctly to findVersions") + void execute_should_pass_pagination_parameters_correctly() throws CommandException { + // Arrange + Integer expectedLimit = 10; + Integer expectedOffset = 20; + when(versionServiceMock.findVersions(anyLong(), any(), any(), anyBoolean(), anyBoolean())) + .thenReturn(Collections.emptyList()); + + GetDatasetVersionSummariesCommand command = new GetDatasetVersionSummariesCommand(requestMock, datasetMock, expectedLimit, expectedOffset); + + // Act + command.execute(contextMock); + + // Assert + verify(versionServiceMock).findVersions( + eq(DATASET_ID), + eq(expectedOffset), + eq(expectedLimit), + anyBoolean(), + eq(true) + ); + } + + @Test + @DisplayName("execute should correctly convert DatasetVersion list to DatasetVersionSummary list") + void execute_should_convert_versions_to_summaries() throws CommandException { + // Arrange: Mock the service to return a list with one version + when(versionServiceMock.findVersions(anyLong(), any(), any(), anyBoolean(), anyBoolean())) + .thenReturn(List.of(versionMock)); + + // Arrange: Prepare a dummy summary object + DatasetVersionSummary dummySummary = new DatasetVersionSummary(1L, "V1", null, null, null, null); + + // Arrange: Mock the static 'from' method to return our dummy summary + try (MockedStatic mockedStatic = Mockito.mockStatic(DatasetVersionSummary.class)) { + mockedStatic.when(() -> DatasetVersionSummary.from(versionMock)) + .thenReturn(Optional.of(dummySummary)); + + GetDatasetVersionSummariesCommand command = new GetDatasetVersionSummariesCommand(requestMock, datasetMock, null, null); + + // Act + List result = command.execute(contextMock); + + // Assert + assertThat(result) + .isNotNull() + .hasSize(1) + .containsExactly(dummySummary); + } + } +} From 7bb99f9ade1bb9d3639a257c21668cd2b0fa9b75 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 2 Oct 2025 14:30:20 -0400 Subject: [PATCH 347/634] #11772 add cleanup to tests --- .../harvard/iq/dataverse/api/DatasetsIT.java | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 1faec141512..dbd685d6718 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -4127,11 +4127,6 @@ public void testUpdateDatasetTerms() throws IOException { UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken).then().assertThat().statusCode(OK.getStatusCode()); UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken).then().assertThat().statusCode(OK.getStatusCode()); - Response getDatasetJsonBeforeUpdate = UtilIT.nativeGet(datasetId, apiToken); - getDatasetJsonBeforeUpdate.prettyPrint(); - getDatasetJsonBeforeUpdate.then().assertThat() - .statusCode(OK.getStatusCode()); - String pathToJsonFile = "src/test/resources/json/update-dataset-terms-access.json"; Response updateTerms = UtilIT.updateDatasetTermsAndAccess(datasetPid, apiToken, pathToJsonFile); updateTerms.prettyPrint(); @@ -4175,18 +4170,12 @@ public void testUpdateDatasetTerms() throws IOException { //Make installation "public install" to see that terms of access may not be set UtilIT.setSetting(SettingsServiceBean.Key.PublicInstall, "true"); - createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); - createDataset.prettyPrint(); - createDataset.then().assertThat() - .statusCode(CREATED.getStatusCode()); - updateTerms = UtilIT.updateDatasetTermsAndAccess(datasetPid, apiToken, pathToJsonFile); updateTerms.prettyPrint(); // should fail because its now a public install updateTerms.then().assertThat() .statusCode(BAD_REQUEST.getStatusCode()) .body("message", equalTo("Setting File Access Request or Terms of Access is not permitted on a public installation.")); - //Setting File Access Request or Terms of Access is not permitted on a public installation. pathToJsonFile = "src/test/resources/json/update-dataset-terms-no-access.json"; @@ -4196,9 +4185,39 @@ public void testUpdateDatasetTerms() throws IOException { updateTerms.then().assertThat() .statusCode(OK.getStatusCode()) .body("data.fileAccessRequest", equalTo(false)); + + String badPID = "QQQAndABatmanSymbol"; + updateTerms = UtilIT.updateDatasetTermsAndAccess(badPID, apiToken, pathToJsonFile); + updateTerms.prettyPrint(); + // should get bad Request Dataset not found.... + updateTerms.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", containsString("QQQ"));; //reset public install UtilIT.setSetting(SettingsServiceBean.Key.PublicInstall, publicInstall); + + String username = UtilIT.getUsernameFromResponse(createUser); + //for cleanup + Response makeSuperUser = UtilIT.setSuperuserStatus(username, true); + + // Clean up + + Response destroyDatasetResponse = UtilIT.destroyDataset(datasetId, apiToken); + destroyDatasetResponse.prettyPrint(); + assertEquals(200, destroyDatasetResponse.getStatusCode()); + + destroyDatasetResponse = UtilIT.destroyDataset(datasetId2, apiToken); + destroyDatasetResponse.prettyPrint(); + assertEquals(200, destroyDatasetResponse.getStatusCode()); + + + Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); + deleteDataverseResponse.prettyPrint(); + assertEquals(200, deleteDataverseResponse.getStatusCode()); + + Response deleteUserResponse = UtilIT.deleteUser(username); + assertEquals(200, deleteUserResponse.getStatusCode()); } From 653b93cfdd052beef09123083ee696ce03f98e70 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 2 Oct 2025 16:18:21 -0400 Subject: [PATCH 348/634] #11772 update command test --- .../impl/UpdateDatasetTermsOfUseCommand.java | 8 ++--- .../UpdateDatasetTermsOfUseCommandTest.java | 34 ++++++++++++++++--- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommand.java index 74aa2ac0138..c5f767a886c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommand.java @@ -1,7 +1,4 @@ -/* - * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license - * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template - */ + package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataset; @@ -12,8 +9,7 @@ import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; -import edu.harvard.iq.dataverse.engine.command.exception.InvalidCommandArgumentsException; -import edu.harvard.iq.dataverse.util.BundleUtil; + /** * diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommandTest.java index e9d357ecbe6..e49b81fe7d7 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommandTest.java @@ -1,19 +1,26 @@ -/* - * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license - * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template - */ + package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.dataset.DatasetFieldsValidator; import edu.harvard.iq.dataverse.engine.DataverseEngine; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.engine.command.exception.InvalidCommandArgumentsException; +import edu.harvard.iq.dataverse.util.BundleUtil; +import java.util.ArrayList; +import java.util.List; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.mockito.Mock; +import org.mockito.Mockito; import static org.mockito.Mockito.when; import org.mockito.MockitoAnnotations; @@ -38,8 +45,9 @@ public class UpdateDatasetTermsOfUseCommandTest { private DatasetVersion datasetVersionMock; @Mock private DatasetFieldsValidator datasetFieldsValidatorMock; + @Mock + private AuthenticatedUser authenticatedUser; - private UpdateDatasetFieldsCommand sut; @BeforeEach public void setUp() throws CommandException { @@ -50,6 +58,22 @@ public void setUp() throws CommandException { when(commandContextMock.datasetFieldsValidator()).thenReturn(datasetFieldsValidatorMock); when(datasetMock.getOrCreateEditVersion()).thenReturn(datasetVersionMock); when(datasetVersionMock.getTermsOfUseAndAccess()).thenReturn(termsOfUseAndAccessStub); + Mockito.when(dataverseRequestStub.getUser()).thenReturn(authenticatedUser); } + @Test + public void execute_shouldThrowException_whenUserIsNotAuthenticated() { + // Arrange + Mockito.when(authenticatedUser.isAuthenticated()).thenReturn(false); + Dataset dataset = new Dataset(); + TermsOfUseAndAccess toua = new TermsOfUseAndAccess(); + UpdateDatasetTermsOfUseCommand sut = new UpdateDatasetTermsOfUseCommand(dataset, toua, dataverseRequestStub); + + // Act & Assert + assertThrows(CommandException.class, () -> { + sut.execute(commandContextMock); + }, BundleUtil.getStringFromBundle("dataverse.link.user")); + } + + } From b5fa671da6380c7524a8e9b703caf266011ad3e9 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 2 Oct 2025 16:58:17 -0400 Subject: [PATCH 349/634] #11772 update command test --- .../UpdateDatasetTermsOfUseCommandTest.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommandTest.java index e49b81fe7d7..e83211ff290 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommandTest.java @@ -21,6 +21,7 @@ import org.junit.jupiter.api.Test; import org.mockito.Mock; import org.mockito.Mockito; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import org.mockito.MockitoAnnotations; @@ -58,21 +59,26 @@ public void setUp() throws CommandException { when(commandContextMock.datasetFieldsValidator()).thenReturn(datasetFieldsValidatorMock); when(datasetMock.getOrCreateEditVersion()).thenReturn(datasetVersionMock); when(datasetVersionMock.getTermsOfUseAndAccess()).thenReturn(termsOfUseAndAccessStub); - Mockito.when(dataverseRequestStub.getUser()).thenReturn(authenticatedUser); + } @Test - public void execute_shouldThrowException_whenUserIsNotAuthenticated() { + public void latestVersionShouldBeDraft() { // Arrange - Mockito.when(authenticatedUser.isAuthenticated()).thenReturn(false); + Dataset dataset = new Dataset(); TermsOfUseAndAccess toua = new TermsOfUseAndAccess(); UpdateDatasetTermsOfUseCommand sut = new UpdateDatasetTermsOfUseCommand(dataset, toua, dataverseRequestStub); + try { + dataset = sut.execute(commandContextMock); + verify(commandContextMock).engine(); + } catch (CommandException ce){ + + } // Act & Assert - assertThrows(CommandException.class, () -> { - sut.execute(commandContextMock); - }, BundleUtil.getStringFromBundle("dataverse.link.user")); + DatasetVersion retVal = dataset.getLatestVersion(); + } From 39626a5e74b893ba30e6756ecb6bb4d54903e97e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Oct 2025 23:12:52 +0000 Subject: [PATCH 350/634] Bump peter-evans/create-or-update-comment from 4 to 5 Bumps [peter-evans/create-or-update-comment](https://github.com/peter-evans/create-or-update-comment) from 4 to 5. - [Release notes](https://github.com/peter-evans/create-or-update-comment/releases) - [Commits](https://github.com/peter-evans/create-or-update-comment/compare/v4...v5) --- updated-dependencies: - dependency-name: peter-evans/create-or-update-comment dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/container_app_pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_app_pr.yml b/.github/workflows/container_app_pr.yml index a5dddc755d1..9ebf3249261 100644 --- a/.github/workflows/container_app_pr.yml +++ b/.github/workflows/container_app_pr.yml @@ -86,7 +86,7 @@ jobs: :ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container). Use by referencing with full name as printed above, mind the registry name. # Leave a note when things have gone sideways - - uses: peter-evans/create-or-update-comment@v4 + - uses: peter-evans/create-or-update-comment@v5 if: ${{ failure() }} with: issue-number: ${{ github.event.client_payload.pull_request.number }} From f612afb22b34db8f8148da499954ef892b413f74 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 3 Oct 2025 10:42:06 -0400 Subject: [PATCH 351/634] #11772 update test --- .../impl/UpdateDatasetTermsOfUseCommandTest.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommandTest.java index e83211ff290..51077b368c3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTermsOfUseCommandTest.java @@ -16,6 +16,7 @@ import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -68,12 +69,13 @@ public void latestVersionShouldBeDraft() { Dataset dataset = new Dataset(); TermsOfUseAndAccess toua = new TermsOfUseAndAccess(); - UpdateDatasetTermsOfUseCommand sut = new UpdateDatasetTermsOfUseCommand(dataset, toua, dataverseRequestStub); + UpdateDatasetTermsOfUseCommand sut = new UpdateDatasetTermsOfUseCommand(dataset, toua, dataverseRequestStub); try { - dataset = sut.execute(commandContextMock); - verify(commandContextMock).engine(); - } catch (CommandException ce){ - + Dataset updatedDataset = sut.execute(commandContextMock); + verify(commandContextMock).engine(); + assertNotNull(datasetMock); + } catch (CommandException ce) { + } // Act & Assert From 89b88b5c7d02a203d15ec877b5d8c3b71571e11d Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 6 Oct 2025 09:33:30 +0100 Subject: [PATCH 352/634] Added: pagination test cases to testSummaryDatasetVersionsDifferencesAPI IT --- .../harvard/iq/dataverse/api/DatasetsIT.java | 60 ++++++++++++++----- .../edu/harvard/iq/dataverse/api/UtilIT.java | 9 ++- 2 files changed, 53 insertions(+), 16 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 5dac4b1d05c..b94f9276a51 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -6374,11 +6374,8 @@ public void testCompareDatasetVersionsAPI() throws InterruptedException { compareResponse = UtilIT.compareDatasetVersions(datasetPersistentId, ":latest-published", ":draft", apiToken, true); compareResponse.prettyPrint(); compareResponse.then().assertThat().statusCode(OK.getStatusCode()); - - - } - + @Test public void testSummaryDatasetVersionsDifferencesAPI() throws InterruptedException { @@ -6477,11 +6474,10 @@ public void testSummaryDatasetVersionsDifferencesAPI() throws InterruptedExcepti Response updateTerms = UtilIT.updateDatasetJsonLDMetadata(datasetId, apiToken, jsonLDTerms, true); updateTerms.then().assertThat() .statusCode(OK.getStatusCode()); - - + Response compareResponse = UtilIT.summaryDatasetVersionDifferences(datasetPersistentId, apiToken); - compareResponse.prettyPrint(); + compareResponse.prettyPrint(); compareResponse.then().assertThat() .body("data[1].versionNumber", equalTo("1.0")) @@ -6498,32 +6494,66 @@ public void testSummaryDatasetVersionsDifferencesAPI() throws InterruptedExcepti .statusCode(OK.getStatusCode()); //user with no privileges will only see the published version - + Response createUsernoPriv = UtilIT.createRandomUser(); assertEquals(200, createUsernoPriv.getStatusCode()); String apiTokenNoPriv = UtilIT.getApiTokenFromResponse(createUsernoPriv); - + Response compareResponse2 = UtilIT.summaryDatasetVersionDifferences(datasetPersistentId, apiTokenNoPriv); compareResponse2.prettyPrint(); compareResponse2.then().assertThat() .body("data[0].versionNumber", CoreMatchers.equalTo("1.0")) .body("data[0].summary", CoreMatchers.equalTo("firstPublished")) .statusCode(OK.getStatusCode()); - + Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, "Test deaccession reason.", null, apiToken); deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode()); - + compareResponse = UtilIT.summaryDatasetVersionDifferences(datasetPersistentId, apiToken); - compareResponse.prettyPrint(); - + compareResponse.prettyPrint(); + compareResponse.then().assertThat() .body("data[1].versionNumber", equalTo("1.0")) .body("data[1].summary.deaccessioned.reason", equalTo("Test deaccession reason.")) .body("data[0].versionNumber", equalTo("DRAFT")) .body("data[0].summary.", equalTo("previousVersionDeaccessioned")) .statusCode(OK.getStatusCode()); - - + + // Pagination tests + + // Publish the current DRAFT as a minor version to create version 2.0 + // This will give us more versions to test pagination against. + + publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPersistentId, "major", apiToken); + publishDataset.then().assertThat().statusCode(OK.getStatusCode()); + + // Make one file change to create a new DRAFT on top of version 2.0 + pathToJsonFilePostPub = "doc/sphinx-guides/source/_static/api/dataset-add-metadata-after-pub.json"; + addDataToPublishedVersion = UtilIT.addDatasetMetadataViaNative(datasetPersistentId, pathToJsonFilePostPub, apiToken); + addDataToPublishedVersion.then().assertThat().statusCode(OK.getStatusCode()); + + // TEST 1: No pagination. Should return all 3 version differences (DRAFT, 2.0, 1.0). + Response compareAllResponse = UtilIT.summaryDatasetVersionDifferences(datasetPersistentId, apiToken); + compareAllResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", is(3)) + .body("data[0].versionNumber", equalTo("DRAFT")) + .body("data[1].versionNumber", equalTo("2.0")) + .body("data[2].versionNumber", equalTo("1.0")); + + // TEST 2: Use limit=1. Should return only the latest difference (DRAFT). + Response compareLimitResponse = UtilIT.summaryDatasetVersionDifferences(datasetPersistentId, 1, null, apiToken); + compareLimitResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", is(1)) + .body("data[0].versionNumber", equalTo("DRAFT")); + + // TEST 3: Use limit=1 and offset=1. Should skip the first result and return the second (2.0). + Response compareOffsetResponse = UtilIT.summaryDatasetVersionDifferences(datasetPersistentId, 1, 1, apiToken); + compareOffsetResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.size()", is(1)) + .body("data[0].versionNumber", equalTo("2.0")); } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index e5d3f908349..20c1a016450 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1787,14 +1787,21 @@ static Response compareDatasetVersions(String persistentId, String versionNumber + "&includeDeaccessioned=" + includeDeaccessioned); } + + static Response summaryDatasetVersionDifferences(String persistentId, String apiToken) { + return summaryDatasetVersionDifferences(persistentId, null, null, apiToken); + } - static Response summaryDatasetVersionDifferences(String persistentId, String apiToken) { + static Response summaryDatasetVersionDifferences(String persistentId, Integer limit, Integer offset, String apiToken) { return given() .header(API_TOKEN_HTTP_HEADER, apiToken) + .queryParam("limit", limit) + .queryParam("offset", offset) .get("/api/datasets/:persistentId/versions/compareSummary" + "?persistentId=" + persistentId); } + static Response getDatasetWithOwners(String persistentId, String apiToken, boolean returnOwners) { return given() .header(API_TOKEN_HTTP_HEADER, apiToken) From 93823a7b40dcfab8ddca6dd44860cd3305fbabf5 Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 6 Oct 2025 09:38:30 +0100 Subject: [PATCH 353/634] Added: pagination explanation docs to compareSummary datasets endpoint --- doc/sphinx-guides/source/api/native-api.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 19a4d79c5ae..6fe0147a683 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2107,6 +2107,16 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/:persistentId/versions/compareSummary?persistentId=doi:10.5072/FK2/BCCP9Z" +You can control pagination of the results using the following optional query parameters. + +* ``limit``: The maximum number of version differences to return. +* ``offset``: The number of version differences to skip from the beginning of the list. Used for retrieving subsequent pages of results. + +For example, to get the second page of results, with 2 items per page, you would use ``limit=2`` and ``offset=2`` (skipping the first two results). + +.. code-block:: bash + + curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/:persistentId/versions/compareSummary?persistentId=doi:10.5072/FK2/BCCP9Z&limit=2&offset=2" Update Metadata For a Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From e043bf16f5c04f7a672e5b8f39c77ae1bdebd508 Mon Sep 17 00:00:00 2001 From: jo-pol Date: Mon, 6 Oct 2025 15:32:16 +0200 Subject: [PATCH 354/634] restored suppressing dataverse field with getDatasetCount rather than findAll --- .../edu/harvard/iq/dataverse/DatasetPage.java | 8 ++++++++ .../iq/dataverse/DataverseConverter.java | 17 ++++++++++++++++- src/main/webapp/dataset.xhtml | 2 +- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index b41e8d4ac35..da1345123b3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -105,6 +105,8 @@ import jakarta.persistence.OptimisticLockException; import org.apache.commons.lang3.StringUtils; +import org.joda.time.DateTime; +import org.joda.time.Duration; import org.primefaces.event.FileUploadEvent; import org.primefaces.model.file.UploadedFile; @@ -332,6 +334,7 @@ public enum DisplayMode { private List linkingDVSelectItems; private Dataverse linkingDataverse; private Dataverse selectedHostDataverse; + private boolean hasDataversesToChoose; public Dataverse getSelectedHostDataverse() { return selectedHostDataverse; @@ -1781,6 +1784,11 @@ public void setDataverseTemplates(List